fragments as source

CHANGELOG.md

@@ -10,6 +10,7 @@
   * [Ecto.Query] Support select aliases through `selected_as/1` and `selected_as/2`
   * [Ecto.Query] Allow `parent_as/1` in `type/2`
   * [Ecto.Query] Add `with_named_binding/3`
+  * [Ecto.Query] Allow fragment sources in keyword queries
   * [Ecto.Repo] Support `idle_interval` query parameter in connection URL
   * [Ecto.Repo] Log human-readable UUIDs by using pre-dumped query parameters
 

lib/ecto/query.ex

@@ -385,7 +385,7 @@ defmodule Ecto.Query do
 
   defmodule FromExpr do
     @moduledoc false
-    defstruct [:source, :as, :prefix, hints: []]
+    defstruct [:source, :file, :line, :as, :prefix, params: [], hints: []]
   end
 
   defmodule DynamicExpr do
@@ -779,8 +779,8 @@ defmodule Ecto.Query do
   It can either be a keyword query or a query expression.
 
   If it is a keyword query the first argument must be
-  either an `in` expression, or a value that implements
-  the `Ecto.Queryable` protocol. If the query needs a
+  either an `in` expression, a value that implements
+  the `Ecto.Queryable` protocol, or an `Ecto.Query.API.fragment/1`. If the query needs a
   reference to the data source in any other part of the
   expression, then an `in` must be used to create a reference
   variable. The second argument should be a keyword query
@@ -791,14 +791,31 @@ defmodule Ecto.Query do
   a value that implements the `Ecto.Queryable` protocol
   and the second argument the expression.
 
-  ## Keywords example
+  ## Keywords examples
 
+      # `in` expression
       from(c in City, select: c)
 
-  ## Expressions example
+      # Ecto.Queryable
+      from(City, limit: 1)
 
+      # Fragment
+      from(f in fragment("generate_series(?, ?) as x", ^0, ^100000), select f.x)
+
+  ## Expressions examples
+
+      # Schema
       City |> select([c], c)
 
+      # Source
+      "cities" |> select([c], c)
+
+      # Source with schema
+      {"cities", Source} |> select([c], c)
+
+      # Ecto.Query
+      from(c in Cities) |> select([c], c)
+
   ## Examples
 
       def paginate(query, page, size) do

lib/ecto/query/builder/from.ex

@@ -35,13 +35,22 @@ defmodule Ecto.Query.Builder.From do
   """
   @spec escape(Macro.t(), Macro.Env.t()) :: {Macro.t(), Keyword.t()}
   def escape({:in, _, [var, query]}, env) do
+    query = escape_source(query, env)
     Builder.escape_binding(query, List.wrap(var), env)
   end
 
-  def escape(query, _env) do
+  def escape(query, env) do
+    query = escape_source(query, env)
     {query, []}
   end
 
+  defp escape_source({:fragment, _, _} = fragment, env) do
+    {fragment, {params, _acc}} = Builder.escape(fragment, :any, {[], %{}}, [], env)
+    {fragment, Builder.escape_params(params)}
+  end
+
+  defp escape_source(query, _env), do: query
+
   @doc """
   Builds a quoted expression.
 
@@ -81,29 +90,34 @@ defmodule Ecto.Query.Builder.From do
         # dependencies between modules are added
         source = quote(do: unquote(schema).__schema__(:source))
         {:ok, prefix} = prefix || {:ok, quote(do: unquote(schema).__schema__(:prefix))}
-        {query(prefix, source, schema, as, hints), binds, 1}
+        {query(prefix, {source, schema}, [], as, hints, env.file, env.line), binds, 1}
 
       source when is_binary(source) ->
         {:ok, prefix} = prefix || {:ok, nil}
         # When a binary is used, there is no schema
-        {query(prefix, source, nil, as, hints), binds, 1}
+        {query(prefix, {source, nil}, [], as, hints, env.file, env.line), binds, 1}
 
       {source, schema} when is_binary(source) and is_atom(schema) ->
         {:ok, prefix} = prefix || {:ok, quote(do: unquote(schema).__schema__(:prefix))}
-        {query(prefix, source, schema, as, hints), binds, 1}
+        {query(prefix, {source, schema}, [], as, hints, env.file, env.line), binds, 1}
+
+      {{:{}, _, [:fragment, _, _]} = fragment, params} ->
+        {:ok, prefix} = prefix || {:ok, nil}
+        {query(prefix, fragment, params, as, hints, env.file, env.line), binds, 1}
 
       _other ->
-        quoted = quote do
-          Ecto.Query.Builder.From.apply(unquote(query), unquote(length(binds)), unquote(as), unquote(prefix), unquote(hints))
-        end
+        quoted =
+          quote do
+            Ecto.Query.Builder.From.apply(unquote(query), unquote(length(binds)), unquote(as), unquote(prefix), unquote(hints))
+          end
 
         {quoted, binds, nil}
     end
   end
 
-  defp query(prefix, source, schema, as, hints) do
+  defp query(prefix, source, params, as, hints, file, line) do
     aliases = if as, do: [{as, 0}], else: []
-    from_fields = [source: {source, schema}, as: as, prefix: prefix, hints: hints]
+    from_fields = [source: source, params: params, as: as, prefix: prefix, hints: hints, file: file, line: line]
 
     query_fields = [
       from: {:%, [], [Ecto.Query.FromExpr, {:%{}, [], from_fields}]},

lib/ecto/query/inspect.ex

@@ -89,7 +89,7 @@ defimpl Inspect, for: Ecto.Query do
       |> generate_names()
       |> List.to_tuple()
 
-    from = bound_from(query.from, elem(names, 0))
+    from = bound_from(query.from, elem(names, 0), names)
     joins = joins(query.joins, names)
     preloads = preloads(query.preloads)
     assocs = assocs(query.assocs, names)
@@ -128,19 +128,20 @@ defimpl Inspect, for: Ecto.Query do
     ])
   end
 
-  defp bound_from(nil, name), do: ["from #{name} in query"]
+  defp bound_from(nil, name, _names), do: ["from #{name} in query"]
 
-  defp bound_from(%{source: source} = from, name) do
-    ["from #{name} in #{inspect_source(source)}"] ++ kw_as_and_prefix(from)
+  defp bound_from(from, name, names) do
+    ["from #{name} in #{inspect_source(from, names)}"] ++ kw_as_and_prefix(from)
   end
 
-  defp inspect_source(%Ecto.Query{} = query), do: "^" <> inspect(query)
-  defp inspect_source(%Ecto.SubQuery{query: query}), do: "subquery(#{to_string(query)})"
-  defp inspect_source({source, nil}), do: inspect(source)
-  defp inspect_source({nil, schema}), do: inspect(schema)
+  defp inspect_source(%{source: %Ecto.Query{} = query}, _names), do: "^" <> inspect(query)
+  defp inspect_source(%{source: %Ecto.SubQuery{query: query}}, _names), do: "subquery(#{to_string(query)})"
+  defp inspect_source(%{source: {source, nil}}, _names), do: inspect(source)
+  defp inspect_source(%{source: {nil, schema}}, _names), do: inspect(schema)
+  defp inspect_source(%{source: {:fragment, _, _} = source} = part, names), do: "#{expr(source, names, part)}"
 
-  defp inspect_source({source, schema} = from) do
-    inspect(if source == schema.__schema__(:source), do: schema, else: from)
+  defp inspect_source(%{source: {source, schema}}, _names) do
+    inspect(if source == schema.__schema__(:source), do: schema, else: {source, schema})
   end
 
   defp joins(joins, names) do
@@ -154,17 +155,8 @@ defimpl Inspect, for: Ecto.Query do
     [{join_qual(qual), string}] ++ kw_as_and_prefix(join) ++ maybe_on(on, names)
   end
 
-  defp join(
-         %JoinExpr{qual: qual, source: {:fragment, _, _} = source, on: on} = join = part,
-         name,
-         names
-       ) do
-    string = "#{name} in #{expr(source, names, part)}"
-    [{join_qual(qual), string}] ++ kw_as_and_prefix(join) ++ [on: expr(on, names)]
-  end
-
-  defp join(%JoinExpr{qual: qual, source: source, on: on} = join, name, names) do
-    string = "#{name} in #{inspect_source(source)}"
+  defp join(%JoinExpr{qual: qual, on: on} = join, name, names) do
+    string = "#{name} in #{inspect_source(join, names)}"
     [{join_qual(qual), string}] ++ kw_as_and_prefix(join) ++ [on: expr(on, names)]
   end
 
@@ -373,6 +365,7 @@ defimpl Inspect, for: Ecto.Query do
   defp from_sources(%Ecto.SubQuery{query: query}), do: from_sources(query.from.source)
   defp from_sources({source, schema}), do: schema || source
   defp from_sources(nil), do: "query"
+  defp from_sources({:fragment, _, _}), do: "fragment"
 
   defp join_sources(joins) do
     joins

lib/ecto/query/planner.ex

@@ -234,6 +234,11 @@ defmodule Ecto.Query.Planner do
     error!(query, "query must have a from expression")
   end
 
+  defp plan_from(%{from: %{source: {:fragment, _, _}}, preloads: preloads, assocs: assocs} = query, _adapter)
+       when assocs != [] or preloads != [] do
+    error!(query, "cannot preload associations with a fragment source")
+  end
+
   defp plan_from(%{from: from} = query, adapter) do
     plan_source(query, from, adapter)
   end
@@ -258,7 +263,7 @@ defmodule Ecto.Query.Planner do
        do: {expr, source}
 
   defp plan_source(query, %{source: {:fragment, _, _}, prefix: prefix} = expr, _adapter),
-       do: error!(query, expr, "cannot set prefix: #{inspect(prefix)} option for fragment joins")
+       do: error!(query, expr, "cannot set prefix: #{inspect(prefix)} option for fragment sources")
 
   defp plan_subquery(subquery, query, prefix, adapter, source?) do
     %{query: inner_query} = subquery
@@ -631,9 +636,10 @@ defmodule Ecto.Query.Planner do
     {query, params, finalize_cache(query, operation, cache)}
   end
 
-  defp merge_cache(:from, _query, from, {cache, params}, _operation, _adapter) do
+  defp merge_cache(:from, query, from, {cache, params}, _operation, adapter) do
     {key, params} = source_cache(from, params)
-    {merge_cache({:from, key, from.hints}, cache, key != :nocache), params}
+    {params, source_cacheable?} = cast_and_merge_params(:from, query, from, params, adapter)
+    {merge_cache({:from, key, from.hints}, cache, source_cacheable? and key != :nocache), params}
   end
 
   defp merge_cache(kind, query, expr, {cache, params}, _operation, adapter)
@@ -910,6 +916,9 @@ defmodule Ecto.Query.Planner do
   def ensure_select(%{select: nil, from: %{source: {_, nil}}} = query, true) do
     error! query, "queries that do not have a schema need to explicitly pass a :select clause"
   end
+  def ensure_select(%{select: nil, from: %{source: {:fragment, _, _}}} = query, true) do
+    error! query, "queries from a fragment need to explicitly pass a :select clause"
+  end
   def ensure_select(%{select: nil} = query, true) do
     %{query | select: %SelectExpr{expr: {:&, [], [0]}, line: __ENV__.line, file: __ENV__.file}}
   end

test/ecto/query/inspect_test.exs

@@ -299,14 +299,26 @@ defmodule Ecto.Query.InspectTest do
 
   test "fragments" do
     value = "foobar"
+
     assert i(from(x in Post, where: fragment("downcase(?) == ?", x.id, ^value))) ==
-           ~s{from p0 in Inspect.Post, where: fragment("downcase(?) == ?", p0.id, ^"foobar")}
+             ~s{from p0 in Inspect.Post, where: fragment("downcase(?) == ?", p0.id, ^"foobar")}
 
     assert i(from(x in Post, where: fragment(^[title: [foo: "foobar"]]))) ==
-           ~s{from p0 in Inspect.Post, where: fragment(title: [foo: "foobar"])}
+             ~s{from p0 in Inspect.Post, where: fragment(title: [foo: "foobar"])}
 
     assert i(from(x in Post, where: fragment(title: [foo: ^value]))) ==
-      ~s{from p0 in Inspect.Post, where: fragment(title: [foo: ^"foobar"])}
+             ~s{from p0 in Inspect.Post, where: fragment(title: [foo: ^"foobar"])}
+
+    assert i(from(x in fragment("SELECT ? as title", ^value))) ==
+             ~s{from f0 in fragment("SELECT ? as title", ^"foobar")}
+
+    assert i(
+             from(x in fragment("SELECT ? as title", ^value),
+               join: y in fragment("SELECT ? as title", ^value),
+               on: x.title == y.title
+             )
+           ) ==
+             ~s{from f0 in fragment("SELECT ? as title", ^"foobar"), join: f1 in fragment(\"SELECT ? as title\", ^\"foobar\"), on: f0.title == f1.title}
   end
 
   test "json_extract_path" do

test/ecto/query/planner_test.exs

@@ -127,27 +127,30 @@ defmodule Ecto.Query.PlannerTest do
     subquery = from Comment, where: [text: ^"subquery"]
 
     query =
-      from p in Post,
-        select: {p.title, ^"select"},
+      from f in fragment("SELECT ? <>  ? as title", ^"fragment_source1", ^"fragment_source2"),
+        select: {f.title, ^"select"},
         join: c in subquery(subquery),
         on: c.text == ^"join",
+        join: p in Post,
+        on: f.title == p.title,
         left_join: d in assoc(p, :comments),
         union_all: ^union,
         windows: [foo: [partition_by: fragment("?", ^"windows")]],
-        where: p.title == ^"where",
-        group_by: p.title == ^"group_by",
-        having: p.title == ^"having",
+        where: f.title == ^"where",
+        group_by: f.title == ^"group_by",
+        having: f.title == ^"having",
         order_by: [asc: fragment("?", ^"order_by")],
         limit: ^0,
         offset: ^1
 
     {_query, cast_params, dump_params, _key} = plan(query)
+
     assert cast_params ==
-             ["select", "subquery", "join", "where", "group_by", "having", "windows"] ++
+             ["select", "fragment_source1", "fragment_source2", "subquery", "join", "where", "group_by", "having", "windows"] ++
                ["union", "order_by", 0, 1]
 
     assert dump_params ==
-             ["select", "subquery", "join", "where", "group_by", "having", "windows"] ++
+             ["select", "fragment_source1", "fragment_source2", "subquery", "join", "where", "group_by", "having", "windows"] ++
                ["union", "order_by", 0, 1]
   end
 
@@ -157,6 +160,12 @@ defmodule Ecto.Query.PlannerTest do
     end
   end
 
+  test "plan: fragment from cannot have preloads" do
+    assert_raise Ecto.QueryError, ~r"cannot preload associations with a fragment source", fn ->
+      plan(from f in fragment("select 1"), preload: :field)
+    end
+  end
+
   test "plan: casts values" do
     {_query, cast_params, dump_params, _key} = plan(Post |> where([p], p.id == ^"1"))
     assert cast_params == [1]

test/ecto/query/subquery_test.exs

@@ -285,6 +285,13 @@ defmodule Ecto.Query.SubqueryTest do
         plan(from(subquery(query), []))
       end
     end
+
+    test "raises on fragment source without :select" do
+      query = from f in fragment("select 1 as x")
+      assert_raise Ecto.SubQueryError, ~r/queries from a fragment need to explicitly pass a :select clause in query/, fn ->
+        plan(from(subquery(query), []))
+      end
+    end
   end
 
   describe "plan: where in subquery" do

test/ecto/query_test.exs

@@ -551,9 +551,7 @@ defmodule Ecto.QueryTest do
       # queries need to be on the same line or == won't work
       assert from(p in "posts", select: 1 < 2) == from(p in "posts", []) |> select([p], 1 < 2)
       assert from(p in "posts", where: 1 < 2)  == from(p in "posts", []) |> where([p], 1 < 2)
-
-      query = "posts"
-      assert (query |> select([p], p.title)) == from(p in query, select: p.title)
+      assert (from(p in "posts") |> select([p], p.title)) == from(p in "posts", select: p.title)
     end
 
     test "are built at compile time with binaries" do