Read through a Backpex.Query

guides/authorization/live-resource-authorization.md

@@ -132,7 +132,7 @@ Backpex.Resource.update_all(item.posts, [set: [user_id: nil]], socket.assigns, M
 
 ### Reads are not gated in `Backpex.Resource`
 
-`Backpex.Resource.list/4`, `get/4`, `count/4` and `reload/4` do not call `can?/3`. Row-level read filtering belongs in [`item_query/3`](item-query.html) — dropping rows after pagination would corrupt item counts and select-all. `:index` and `:show` are enforced when the view mounts. `reload/4` is the read the item action gates use before they authorize; it is the gate that follows it, not the read, that raises.
+`Backpex.Resource.read/1`, `get/4`, `count/1` and `reload/4` do not call `can?/3`. Row-level read filtering belongs in `c:Backpex.LiveResource.scope/1` or [`item_query/3`](item-query.html) — dropping rows after pagination would corrupt item counts and select-all. `:index` and `:show` are enforced when the view mounts. `reload/4` is the read the item action gates use before they authorize; it is the gate that follows it, not the read, that raises.
 
 ### Calling the checks yourself
 

guides/live_resource/item-query.md

@@ -3,6 +3,20 @@
 It is possible to manipulate the query when fetching resources for `index`, `show` and `edit` view when using
 `Backpex.Adapters.Ecto`.
 
+> #### Scope {: .tip}
+>
+> To limit the items of a resource, e.g. to the posts of the current user, prefer the `c:Backpex.LiveResource.scope/1`
+> callback. It returns an expression (see `Backpex.Expr`) and works with every adapter:
+>
+> ```elixir
+> import Backpex.Expr
+>
+> @impl Backpex.LiveResource
+> def scope(assigns), do: expr(user_id == ^assigns.current_user.id)
+> ```
+>
+> Use `item_query` for what only an Ecto query can do, e.g. joins.
+
 In all queries we define a `from` query with a named binding to fetch all existing resources on `index` view or a specific resource on `show` / `edit` view.
 After that, we call the `item_query` function. By default it returns the incoming query.
 

guides/upgrading/v0.25.md

@@ -85,7 +85,7 @@ def condition({field, _config}, value, _assigns), do: expr(ref(^field) >= ^value
 ```
 
 Code that calls the callbacks of filters directly, e.g. a custom adapter, passes the filter as well. The filters in the
-`:filter_configs` criteria are validated. `Backpex.LiveResource.active_filters/1` returns validated filters, too.
+`:filters` assign are validated. `Backpex.LiveResource.active_filters/1` returns validated filters, too.
 
 ## Data access goes through the adapter
 
@@ -172,18 +172,21 @@ def condition({field, _config}, value, _assigns) do
 end
 ```
 
-Tests that call `query/4` of a filter can call `condition/3` and compare the expression, or apply the filter to a query
-with `Backpex.Adapters.Ecto.apply_filters/4`:
+Tests that call `query/4` of a filter can call `condition/3` and compare the expression, or read the items that match
+it with a `Backpex.Query`:
 
 ```elixir
-Post
-|> from(as: ^Backpex.Adapters.Ecto.name_by_schema(Post))
-|> Backpex.Adapters.Ecto.apply_filters(%{status: "draft"}, [status: %{module: MyAppWeb.Filters.Status}], %{})
-|> Repo.all()
+filter = {:status, %{module: MyAppWeb.Filters.Status}}
+
+{:ok, posts} =
+  MyAppWeb.PostLive
+  |> Backpex.Query.new()
+  |> Backpex.Query.filter(Backpex.Filter.to_condition(filter, "draft", %{}))
+  |> Backpex.Resource.read()
 ```
 
-Name the binding as the adapter does, as in the example, if a native expression of the filter uses a named binding such
-as `dynamic([post: p], ...)`. Without it, the query raises with `unknown bind name`.
+The Ecto adapter names the binding of the schema like `Backpex.Adapters.Ecto.name_by_schema/1`, e.g. `:post`, so a
+native expression of the filter can use it, as in `dynamic([post: p], ...)`.
 
 ### Metrics name an aggregate
 
@@ -202,14 +205,13 @@ total_likes: %{module: Backpex.Metrics.Value, label: "Total likes", aggregate: {
 See `Backpex.Metric` for all aggregates. `use BackpexWeb, :metric` no longer imports `Ecto.Query`.
 
 The `query/3` callback of `Backpex.Metric` is gone. A metric module that gets its data in another way implements
-`c:Backpex.Metric.data/3` instead. Backpex calls it instead of the adapter, with the metric, the criteria of the index
-view and its assigns. The criteria hold the search under `:search` and the values and configurations of the active
-filters under `:filter_values` and `:filter_configs`. A metric module with `use BackpexWeb, :metric` that still defines
+`c:Backpex.Metric.data/3` instead. Backpex calls it instead of the adapter, with the metric, the `Backpex.Query` of the
+items of the index view (after the search and the active filters, on all pages) and its assigns. A metric module with `use BackpexWeb, :metric` that still defines
 `query/3` without `data/3` doesn't compile. For other metric modules, Backpex raises when it computes the metric.
 
-`query/3` received a query that the `item_query`, the search and the filters already limited. `data/3` receives the
-criteria instead of a query. To compute the data from the items of the index view, pass the criteria to
-`Backpex.Resource.list/4`. The adapter applies the `item_query`, the search and the filters, as it did for `query/3`:
+`query/3` received an Ecto query that the `item_query`, the search and the filters already limited. `data/3` receives
+a `Backpex.Query` instead. To compute the data from the items of the index view, read it with
+`Backpex.Resource.read/1`. The adapter applies the `item_query`, the search and the filters, as it did for `query/3`:
 
 ```elixir
 # Before
@@ -220,8 +222,8 @@ end
 
 # After
 @impl Backpex.Metric
-def data(_metric, criteria, assigns) do
-  {:ok, orders} = Backpex.Resource.list(criteria, assigns.fields, assigns, assigns.live_resource)
+def data(_metric, query, _assigns) do
+  {:ok, orders} = Backpex.Resource.read(query)
 
   orders |> Enum.map(&MyApp.Orders.total/1) |> Enum.sum()
 end
@@ -237,17 +239,17 @@ def query(_query, _select, _repo), do: MyApp.Orders.open_total()
 
 # After
 @impl Backpex.Metric
-def data(_metric, criteria, assigns) do
-  MyApp.Orders.open_total(assigns.current_user, criteria[:filter_values])
+def data(_metric, _query, assigns) do
+  MyApp.Orders.open_total(assigns.current_user, assigns.filter_values)
 end
 ```
 
 If an aggregate such as `{:sum, :price}` can express the data, use it instead of `data/3`. The adapter computes it in
 the database.
 
-`data/2` of v0.23 is now `c:Backpex.Metric.data/3`. Besides the metric and the assigns, it receives
-the criteria of the index view, which also hold the search. The validated filter values are in
-`criteria[:filter_values]`. A metric module with `use BackpexWeb, :metric` that still defines `data/2` without `data/3`
+`data/2` of v0.23 is now `c:Backpex.Metric.data/3`. Besides the metric and the assigns, it receives the query of the
+index view, which also holds the search (`query.search`). The validated filter values are still in
+`assigns.filter_values`. A metric module with `use BackpexWeb, :metric` that still defines `data/2` without `data/3`
 doesn't compile. For other metric modules, Backpex raises when it computes the metric.
 
 ```elixir
@@ -257,50 +259,51 @@ def data(_metric, assigns), do: MyApp.Orders.open_total(assigns.current_user, as
 
 # After
 @impl Backpex.Metric
-def data(_metric, criteria, assigns), do: MyApp.Orders.open_total(assigns.current_user, criteria[:filter_values])
+def data(_metric, _query, assigns), do: MyApp.Orders.open_total(assigns.current_user, assigns.filter_values)
 ```
 
 ### Custom adapters
 
-If you have a custom adapter, implement the new callbacks:
-
-- `c:Backpex.Adapter.metric/5` returns the data of a metric, the `:aggregate` of the metric for the items that match
-  its `:where` expression. It gets the criteria of `count/4` and ignores `:order`, `:pagination` and `:limit`. Backpex
-  calls `c:Backpex.Metric.data/3` instead if the module of the metric implements it.
+Backpex builds a `Backpex.Query` for every read, and the adapter only runs it. The read callbacks of v0.24 are gone:
+`get/4`, `list/4` and `count/4` no longer exist, and neither do the callbacks this version added at first for metrics
+and the options of association fields. Implement these callbacks instead:
+
+- `c:Backpex.Adapter.read/1` returns the items of a query: the items at the end of its `:from` (the items of the
+  resource, or related items such as the options of an association field), that match its `:filter` and `:search`,
+  sorted by `Backpex.Query.sorts/1`, after its `:offset` and up to its `:limit`, with the relationships of its `:load`.
+  It replaces `get/4` (a filter on the primary key with a limit of 1) and `list/4`.
+- `c:Backpex.Adapter.count/1` returns the number of items of a query.
+- `c:Backpex.Adapter.aggregate/1` returns the values of the `:aggregates` of a query, e.g. the data of metrics. Each
+  aggregate covers the items of the query that also match its `:where` expression.
 - `c:Backpex.Adapter.new_item/2` returns the item for the form of the `:new` action.
 - `c:Backpex.Adapter.describe/2` returns the primary key, the fields with their types and the relationships of the
-  items of the resource, or of related items at the end of a path, see `Backpex.Adapter.Description`. Backpex finds the
-  keys of relationships with it, tells integer fields from others in `Backpex.Filters.Range`, and checks and casts
-  expressions (`Backpex.Resource.cast_expr/2`). A relationship includes the `:related_key` its owner key refers to and
-  the `:primary_key` of the related items.
-- `c:Backpex.Adapter.list_options/4` and `c:Backpex.Adapter.count_options/4` return the items an association field can
-  select. `c:Backpex.Adapter.list_options_by_key/4` is optional. The association fields check submitted values with the
-  `:ids` and `:key` criteria, so they must hold even if the `:options_query` of the field limits the items.
+  items of the resource, or of related items at the end of a path, see `Backpex.Adapter.Description`. Backpex checks
+  and casts the expressions of queries with it, finds the keys of relationships and tells integer fields from others
+  in `Backpex.Filters.Range`. A relationship includes the `:related_key` its owner key refers to and the `:primary_key`
+  of the related items.
 - `c:Backpex.Adapter.put_assoc/4` puts associated items into a changeset.
 - `c:Backpex.Adapter.field_config_schema/0` returns the options your adapter adds to fields. It is optional, and
   `use Backpex.Adapter` defines it from its `:field_config_schema` option.
+- `c:Backpex.Adapter.put_field_options/2` is optional. It takes the options your adapter adds to fields into a query,
+  as adapters no longer get the fields of the LiveResource.
 
-These optional callbacks keep features of v0.24 working with your adapter:
-
-- `c:Backpex.Adapter.options_key/3` returns a term that changes when the options of an association field may change.
-  `Backpex.Fields.HasMany` and `Backpex.Fields.HasManyThrough` load their options again when it changes, e.g. when the
-  `:options_query` depends on a value of the form. Without it, they load their options once.
-
-The criteria passed to `list/4` and `count/4` changed:
-
-- Apply the expression of each filter (`Backpex.Filter.to_condition/3`) instead of calling its `query/4` callback. The
-  function returns the result of the `:condition` option of the filter, if given, or of `c:Backpex.Filter.condition/3`.
-  Pass the expression to `Backpex.Resource.cast_expr/2` first to check its fields and cast its values with your
-  `c:Backpex.Adapter.describe/2`.
-  See `Backpex.Expr` for the structs of expressions and `Backpex.Expr.normalize/1` for keyword lists.
-- The searchable fields in the `:search` criteria no longer have a `:queryable` key.
-- The `:order` criteria no longer has a `:schema` key. Its `:field_name` key holds the name of the field that is ordered
-  by, or `nil`. Its `:nulls` key holds the configured value for the primary key, too. `Backpex.Adapters.Ecto` ignores
-  it for the primary key.
+The filters of a query are already checked and cast with your `describe/2`, see `Backpex.Expr` for the structs of
+expressions. Equal queries read the same items (see `Backpex.Query.key/1`), so the options of association fields are
+read once per distinct query and again when their query changes, e.g. when an `:options_filter` depends on a value of
+the form. The association fields check submitted values with a filter on the options query of the field, so a filter
+must also hold for options your adapter limits natively, e.g. with a limit.
 
 `delete_all/2` should return `{:error, :foreign_key_violation}` if an item can't be deleted because other data still
 references it.
 
+### Scoping items
+
+`c:Backpex.LiveResource.scope/1` returns an expression that every item of the resource matches, e.g. the posts of the
+current user. Unlike the `:item_query` of `Backpex.Adapters.Ecto`, it works with every adapter. The new
+`:options_filter` option of `Backpex.Fields.BelongsTo`, `Backpex.Fields.HasMany` and `Backpex.Fields.HasManyThrough`
+does the same for the options of the field, and `:options_sort` sorts them, e.g. `[name: :asc]`, next to the
+`:options_query` of the Ecto adapter.
+
 ### Custom fields
 
 The options and callbacks of fields that only `Backpex.Adapters.Ecto` used moved to it:
@@ -323,22 +326,22 @@ The options and callbacks of fields that only `Backpex.Adapters.Ecto` used moved
 
 If your field calls one of these functions on a built-in field, e.g. `search_condition/3` of `Backpex.Fields.Number`,
 build the condition in your field instead. Code that calls `schema/2` on the module of a field, e.g. to build the
-criteria of an export, raises now. Build the criteria with `Backpex.LiveResource.build_criteria/1` instead:
+query of an export, raises now. Build the query of the index view with `Backpex.LiveResource.index_query/1` instead. It
+takes the fields of the index view of the LiveResource, so it also works with the assigns of a resource action:
 
 ```elixir
-criteria =
+{:ok, items} =
   assigns
-  |> Backpex.LiveResource.build_criteria()
-  |> Keyword.delete(:pagination)
-
-Backpex.Resource.list(criteria, fields, assigns, live_resource)
+  |> Backpex.LiveResource.index_query()
+  |> Backpex.Query.unset([:limit, :offset])
+  |> Backpex.Resource.read()
 ```
 
 `use Backpex.Field` no longer imports `Ecto.Query`. If your field imports it only for the callbacks Backpex used to
 define, the compiler now warns that the import is unused.
 
-The `:options_query` option of association fields receives the query of the adapter, an `Ecto.Query` for the Ecto
-adapter, and the assigns. Its documentation called the second argument `_field` and showed a function with one
+The `:options_query` option of association fields only works with the Ecto adapter. It receives the `Ecto.Query` and
+the assigns. Use `:options_filter` for a filter that works with every adapter. Its documentation called the second argument `_field` and showed a function with one
 argument, but it always took the assigns as the second argument.
 
 ### `Backpex.Fields.BelongsTo` no longer assigns `:queryable`
@@ -392,8 +395,12 @@ end
 
 - `search_options/3` of `Backpex.LiveResource` is now `Backpex.LiveResource.search_options/2`. It no longer takes the
   schema.
-- `apply_criteria/3` of `Backpex.Adapters.Ecto` is now `Backpex.Adapters.Ecto.apply_criteria/4`. It takes the schema as
-  the last argument.
+- `list_query/4`, `apply_search/4`, `apply_filters/4` and `apply_criteria/3` of `Backpex.Adapters.Ecto` are gone.
+  `Backpex.Adapters.Ecto.ecto_query/1` returns the `Ecto.Query` of a `Backpex.Query`, e.g. to stream the items of an
+  export.
+- `Backpex.Resource.list/4`, `Backpex.Resource.count/4` and `Backpex.LiveResource.build_criteria/1` are now
+  `Backpex.Resource.read/1`, `Backpex.Resource.count/1` and `Backpex.LiveResource.index_query/1`, which build and take
+  a `Backpex.Query`.
 - The `query` functions of `Backpex.Filters.Boolean`, `Backpex.Filters.Select`, `Backpex.Filters.MultiSelect` and
   `Backpex.Filters.Range` are now their `c:Backpex.Filter.condition/3` callbacks, which return the expression of the
   filter. `maybe_query/2` of `Backpex.Filters.Boolean` and `do_query/3` of `Backpex.Filters.Range` are gone, and so is

integration_test/adapters/ecto_expressions_test.exs

@@ -7,11 +7,7 @@ defmodule Backpex.Integration.Adapters.EctoExpressionsTest do
   alias Backpex.Test.Post
   alias Backpex.Test.PostLive
   alias Backpex.Test.User
-
-  defmodule ExpressionFilter do
-    @moduledoc false
-    def condition(_field, expression, _assigns), do: expression
-  end
+  alias Backpex.Test.UserLive
 
   setup do
     ann = insert(:user, username: "ann", first_name: "Ann", last_name: "Ann")
@@ -82,7 +78,7 @@ defmodule Backpex.Integration.Adapters.EctoExpressionsTest do
 
     test "join a path once for every expression that uses it" do
       query =
-        filter_query(Post, [
+        filter_query(PostLive, [
           expr(user.username == "ann"),
           expr(user.age == 30 or user.first_name == "Ann")
         ])
@@ -136,15 +132,15 @@ defmodule Backpex.Integration.Adapters.EctoExpressionsTest do
 
   describe "with the LiveResource" do
     test "casts the values to the types of the fields of its description" do
-      assert titles(expr(likes > "100"), PostLive) == ["100% Elixir"]
-      assert titles(expr(user.role == "user" and user.age == "30"), PostLive) == ["100% Elixir"]
+      assert titles(expr(likes > "100")) == ["100% Elixir"]
+      assert titles(expr(user.role == "user" and user.age == "30")) == ["100% Elixir"]
     end
 
     test "raises for unknown fields and paths through relationships to many items" do
-      assert_raise ArgumentError, ~r/unknown field :unknown/, fn -> titles(expr(unknown == 1), PostLive) end
+      assert_raise ArgumentError, ~r/unknown field :unknown/, fn -> titles(expr(unknown == 1)) end
 
       assert_raise ArgumentError, ~r/:tags to be a relationship to one item/, fn ->
-        titles(expr(tags.name == "elixir"), PostLive)
+        titles(expr(tags.name == "elixir"))
       end
     end
 
@@ -158,19 +154,18 @@ defmodule Backpex.Integration.Adapters.EctoExpressionsTest do
     end
   end
 
-  defp titles(expression, live_resource \\ nil) do
-    Post |> filter_query([expression], live_resource) |> Repo.all() |> Enum.map(& &1.title) |> Enum.sort()
+  defp titles(expression) do
+    PostLive |> filter_query([expression]) |> Repo.all() |> Enum.map(& &1.title) |> Enum.sort()
   end
 
   defp usernames(expression) do
-    User |> filter_query([expression]) |> Repo.all() |> Enum.map(& &1.username) |> Enum.sort()
+    UserLive |> filter_query([expression]) |> Repo.all() |> Enum.map(& &1.username) |> Enum.sort()
   end
 
-  defp filter_query(schema, expressions, live_resource \\ nil) do
-    names = Enum.take([:first, :second, :third], length(expressions))
-    configs = Enum.map(names, &{&1, %{module: ExpressionFilter}})
-    query = from(schema)
-
-    EctoAdapter.apply_filters(query, names |> Enum.zip(expressions) |> Map.new(), configs, %{}, live_resource)
+  defp filter_query(live_resource, expressions) do
+    live_resource
+    |> Backpex.Query.new()
+    |> then(&Enum.reduce(expressions, &1, fn expression, query -> Backpex.Query.filter(query, expression) end))
+    |> EctoAdapter.ecto_query()
   end
 end

integration_test/index/metrics_test.exs

@@ -13,4 +13,16 @@ defmodule Backpex.Integration.Index.MetricsTest do
     |> assert_has(".card-body", text: "Total likes 1600 likes", exact: true)
     |> assert_has(".card-body", text: "Published Posts 2", exact: true)
   end
+
+  test "ignores the filters of :ignore_filters", %{conn: conn} do
+    insert(:post, likes: 250, published: true)
+    insert(:post, likes: 1000, published: false)
+
+    # The published filter is active by default.
+    conn
+    |> visit(~p"/admin/posts")
+    |> assert_has("table tbody tr", count: 1)
+    |> assert_has(".card-body", text: "Total likes 250 likes", exact: true)
+    |> assert_has(".card-body", text: "All posts 2", exact: true)
+  end
 end