introduce CTE + RECURSIVE

pydal/adapters/base.py

@@ -668,7 +668,18 @@ def _select_wcols(
         cache=None,
         cacheable=None,
         processor=None,
+        cte_collector=None,
     ):
+        if cte_collector is None:
+            cte_collector = dict(
+                stack = [],
+                seen = set(),
+                is_recursive = False
+            )
+            is_toplevel = True
+        else:
+            is_toplevel = False
+
         #: parse tablemap
         tablemap = self.tables(query)
         #: apply common filters if needed
@@ -682,6 +693,7 @@ def _select_wcols(
             tablemap.pop(item, None)
         if len(tablemap) < 1:
             raise SyntaxError("Set: no tables selected")
+
         query_tables = list(tablemap)
         #: check for_update argument
         # [Note - gi0baro] I think this should be removed since useless?
@@ -723,27 +735,27 @@ def _select_wcols(
         if join and not left:
             cross_joins = iexcluded + list(itables_to_merge)
             tokens = [table_alias(cross_joins[0])]
-            tokens += [
+            tokens.extend([
                 self.dialect.cross_join(table_alias(t), query_env)
                 for t in cross_joins[1:]
-            ]
-            tokens += [self.dialect.join(t, query_env) for t in ijoin_on]
+            ])
+            tokens.extend([self.dialect.join(t, query_env) for t in ijoin_on])
             sql_t = " ".join(tokens)
         elif not join and left:
             cross_joins = excluded + list(tables_to_merge)
             tokens = [table_alias(cross_joins[0])]
-            tokens += [
+            tokens.extend([
                 self.dialect.cross_join(table_alias(t), query_env)
                 for t in cross_joins[1:]
-            ]
+            ])
             # FIXME: WTF? This is not correct syntax at least on PostgreSQL
             if join_tables:
                 tokens.append(
                     self.dialect.left_join(
                         ",".join([table_alias(t) for t in join_tables]), query_env
                     )
                 )
-            tokens += [self.dialect.left_join(t, query_env) for t in join_on]
+            tokens.extend([self.dialect.left_join(t, query_env) for t in join_on])
             sql_t = " ".join(tokens)
         elif join and left:
             all_tables_in_query = set(
@@ -754,19 +766,19 @@ def _select_wcols(
                 all_tables_in_query.difference(tables_in_joinon)
             )
             tokens = [table_alias(tables_not_in_joinon[0])]
-            tokens += [
+            tokens.extend([
                 self.dialect.cross_join(table_alias(t), query_env)
                 for t in tables_not_in_joinon[1:]
-            ]
-            tokens += [self.dialect.join(t, query_env) for t in ijoin_on]
+            ])
+            tokens.extend([self.dialect.join(t, query_env) for t in ijoin_on])
             # FIXME: WTF? This is not correct syntax at least on PostgreSQL
             if join_tables:
                 tokens.append(
                     self.dialect.left_join(
                         ",".join([table_alias(t) for t in join_tables]), query_env
                     )
                 )
-            tokens += [self.dialect.left_join(t, query_env) for t in join_on]
+            tokens.extend([self.dialect.left_join(t, query_env) for t in join_on])
             sql_t = " ".join(tokens)
         else:
             sql_t = ", ".join(table_alias(t) for t in query_tables)
@@ -804,12 +816,22 @@ def _select_wcols(
                     for t in query_tables
                     if not isinstance(tablemap[t], Select)
                     for x in (
-                        hasattr(tablemap[t], "_primarykey")
-                        and tablemap[t]._primarykey
+                        getattr(tablemap[t], "_primarykey", None)
                         or ["_id"]
                     )
                 ]
             )
+
+        #: build CTE
+        [t.cte(cte_collector) for t in tablemap.values() if getattr(t, 'is_cte', None)]
+        if is_toplevel and cte_collector['stack']:
+            with_cte = [
+                cte_collector['is_recursive'],
+                ', '.join(cte_collector['stack'])
+            ]
+        else:
+            with_cte = None
+
         #: build sql using dialect
         return (
             colnames,
@@ -823,6 +845,7 @@ def _select_wcols(
                 limitby,
                 distinct,
                 for_update and self.can_select_for_update,
+                with_cte
             ),
         )
 

pydal/dialects/base.py

@@ -166,6 +166,25 @@ def delete(self, table, where=None):
             whr = " %s" % self.where(where)
         return "DELETE FROM %s%s;" % (tablename, whr)
 
+
+    def cte(self, tname, fields, sql, recursive = None):
+        '''
+        recursive:list = [union_type, recursive_sql]
+        '''
+        if recursive:
+            r_sql_parts = ['%s %s'%(union, sql) for union, sql in recursive]
+            recursive = ' '.join(r_sql_parts)
+            cte_select = '{select} {recursive}'
+        else:
+            cte_select = '{select}'
+
+        return ('{tname}({fields}) AS (%s)' % cte_select).format(
+            tname = tname,
+            fields = fields,
+            select = sql,
+            recursive = recursive
+        )
+
     def select(
         self,
         fields,
@@ -177,6 +196,7 @@ def select(
         limitby=None,
         distinct=False,
         for_update=False,
+        with_cte=None  # ['recursive' | '', sql]
     ):
         dst, whr, grp, order, limit, offset, upd = "", "", "", "", "", "", ""
         if distinct is True:
@@ -197,7 +217,14 @@ def select(
             offset = " OFFSET %i" % lmin
         if for_update:
             upd = " FOR UPDATE"
-        return "SELECT%s %s FROM %s%s%s%s%s%s%s;" % (
+        if with_cte:
+            recursive, cte = with_cte
+            recursive = ' RECURSIVE' if recursive else ''
+            with_cte = "WITH%s %s " % (recursive, cte)
+        else:
+            with_cte = ""
+        return "%sSELECT%s %s FROM %s%s%s%s%s%s%s;" % (
+            with_cte,
             dst,
             fields,
             tables,

pydal/dialects/db2.py

@@ -81,6 +81,7 @@ def select(
         limitby=None,
         distinct=False,
         for_update=False,
+        with_cte=None,
     ):
         dst, whr, grp, order, limit, offset, upd = "", "", "", "", "", "", ""
         if distinct is True:
@@ -100,7 +101,16 @@ def select(
             limit = " FETCH FIRST %i ROWS ONLY" % lmax
         if for_update:
             upd = " FOR UPDATE"
-        return "SELECT%s %s FROM %s%s%s%s%s%s%s;" % (
+
+        if with_cte:
+            recursive, cte = with_cte
+            recursive = ' RECURSIVE' if recursive else ''
+            with_cte = "WITH%s %s " % (recursive, cte)
+        else:
+            with_cte = ""
+
+        return "%sSELECT%s %s FROM %s%s%s%s%s%s%s;" % (
+            with_cte,
             dst,
             fields,
             tables,

pydal/dialects/firebird.py

@@ -96,6 +96,7 @@ def select(
         limitby=None,
         distinct=False,
         for_update=False,
+        with_cte=None,
     ):
         dst, whr, grp, order, limit, offset, upd = "", "", "", "", "", "", ""
         if distinct is True:
@@ -116,7 +117,16 @@ def select(
             offset = " SKIP %i" % lmin
         if for_update:
             upd = " FOR UPDATE"
-        return "SELECT%s%s%s %s FROM %s%s%s%s%s;" % (
+
+        if with_cte:
+            recursive, cte = with_cte
+            recursive = ' RECURSIVE' if recursive else ''
+            with_cte = "WITH%s %s " % (recursive, cte)
+        else:
+            with_cte = ""
+
+        return "%sSELECT%s%s%s %s FROM %s%s%s%s%s;" % (
+            with_cte,
             dst,
             limit,
             offset,

pydal/dialects/mssql.py

@@ -136,6 +136,7 @@ def select(
         limitby=None,
         distinct=False,
         for_update=False,
+        with_cte=None  # ['recursive' | '', sql]
     ):
         dst, whr, grp, order, limit, upd = "", "", "", "", "", ""
         if distinct is True:
@@ -155,7 +156,16 @@ def select(
             limit = " TOP %i" % lmax
         if for_update:
             upd = " FOR UPDATE"
-        return "SELECT%s%s %s FROM %s%s%s%s%s;" % (
+
+        if with_cte:
+            recursive, cte = with_cte
+            recursive = ' RECURSIVE' if recursive else ''
+            with_cte = "WITH%s %s " % (recursive, cte)
+        else:
+            with_cte = ""
+
+        return "%sSELECT%s%s %s FROM %s%s%s%s%s;" % (
+            with_cte,
             dst,
             limit,
             fields,
@@ -359,6 +369,7 @@ def select(
         limitby=None,
         distinct=False,
         for_update=False,
+        with_cte=None  # ['recursive' | '', sql]
     ):
         dst, whr, grp, order, limit, offset, upd = "", "", "", "", "", "", ""
         if distinct is True:
@@ -383,7 +394,16 @@ def select(
                 )
         if for_update:
             upd = " FOR UPDATE"
-        return "SELECT%s %s FROM %s%s%s%s%s%s%s;" % (
+
+        if with_cte:
+            recursive, cte = with_cte
+            recursive = ' RECURSIVE' if recursive else ''
+            with_cte = "WITH%s %s " % (recursive, cte)
+        else:
+            with_cte = ""
+
+        return "%sSELECT%s %s FROM %s%s%s%s%s%s%s;" % (
+            with_cte,
             dst,
             fields,
             tables,
@@ -409,6 +429,7 @@ def select(
         limitby=None,
         distinct=False,
         for_update=False,
+        with_cte=None  # ['recursive' | '', sql]
     ):
         dst, whr, grp, order, limit, offset, upd = "", "", "", "", "", "", ""
         if distinct is True:
@@ -436,7 +457,16 @@ def select(
                 )
         if for_update:
             upd = " FOR UPDATE"
-        return "SELECT%s %s FROM %s%s%s%s%s%s%s;" % (
+
+        if with_cte:
+            recursive, cte = with_cte
+            recursive = ' RECURSIVE' if recursive else ''
+            with_cte = "WITH%s %s " % (recursive, cte)
+        else:
+            with_cte = ""
+
+        return "%sSELECT%s %s FROM %s%s%s%s%s%s%s;" % (
+            with_cte,
             dst,
             fields,
             tables,

pydal/dialects/oracle.py

@@ -195,6 +195,7 @@ def select(
         limitby=None,
         distinct=False,
         for_update=False,
+        with_cte=None
     ):
         dst, whr, grp, order, limit, offset, upd = "", "", "", "", "", "", ""
         if distinct is True:
@@ -209,19 +210,28 @@ def select(
                 grp += " HAVING %s" % having
         if orderby:
             order = " ORDER BY %s" % orderby
+
+        if with_cte:
+            recursive, cte = with_cte
+            recursive = ' RECURSIVE' if recursive else ''
+            with_cte = "WITH%s %s " % (recursive, cte)
+        else:
+            with_cte = ""
+
         if limitby:
             (lmin, lmax) = limitby
             if whr:
                 whr2 = whr + " AND w_row > %i" % lmin
             else:
                 whr2 = self.where("w_row > %i" % lmin)
             return """
-                SELECT%s * FROM (
+                %sSELECT%s * FROM (
                     SELECT w_tmp.*, ROWNUM w_row FROM (
                         SELECT %s FROM %s%s%s%s
-                    ) w_tmp 
+                    ) w_tmp
                 ) WHERE w_row<=%i and w_row>%i
             """ % (
+                with_cte,
                 dst,
                 fields,
                 tables,
@@ -233,7 +243,8 @@ def select(
             )
         if for_update:
             upd = " FOR UPDATE"
-        return "SELECT%s %s FROM %s%s%s%s%s%s%s;" % (
+        return "%sSELECT%s %s FROM %s%s%s%s%s%s%s;" % (
+            with_cte,
             dst,
             fields,
             tables,

pydal/dialects/sqlite.py

@@ -44,6 +44,7 @@ def select(
         limitby=None,
         distinct=False,
         for_update=False,
+        with_cte= ""
     ):
         if distinct and distinct is not True:
             raise SyntaxError("DISTINCT ON is not supported by SQLite")
@@ -57,6 +58,7 @@ def select(
             limitby,
             distinct,
             for_update,
+            with_cte
         )
 
     def truncate(self, table, mode=""):

pydal/dialects/teradata.py

@@ -86,6 +86,7 @@ def select(
         limitby=None,
         distinct=False,
         for_update=False,
+        with_cte=None,
     ):
         dst, whr, grp, order, limit, offset, upd = "", "", "", "", "", "", ""
         if distinct is True:
@@ -105,7 +106,16 @@ def select(
             limit = " TOP %i" % lmax
         if for_update:
             upd = " FOR UPDATE"
-        return "SELECT%s%s %s FROM %s%s%s%s%s%s;" % (
+
+        if with_cte:
+            recursive, cte = with_cte
+            recursive = ' RECURSIVE' if recursive else ''
+            with_cte = "WITH%s %s " % (recursive, cte)
+        else:
+            with_cte = ""
+
+        return "%sSELECT%s%s %s FROM %s%s%s%s%s%s;" % (
+            with_cte,
             dst,
             limit,
             fields,