Metric filters, expression values, options sort and shared aggregate joins

guides/fields/computed-fields.md

@@ -33,7 +33,9 @@ end
 ```
 
 A value can be a field of the items or of related items (e.g. `expr(author.name)`), a condition (e.g.
-`expr(likes > 0)`) or a string joined with `concat/2`, which leaves out `nil`. We recommend to show computed values on
+`expr(likes > 0)`), a string joined with `concat/2`, which leaves out `nil`, a number computed with `+`, `-`, `*` and
+`/` (e.g. `expr(datetime_diff(ends_at, starts_at, :minute) - pause)`) or a value as a string with `to_string/1`. See
+the [Values](`m:Backpex.Expr#module-values`) of `Backpex.Expr`. We recommend to show computed values on
 the `index` and `show` views only.
 
 > #### Important {: .info}
@@ -61,7 +63,8 @@ end
 
 The path can go through several relationships, e.g. `[:posts, :tags]`, and the value through relationships to one item,
 e.g. `expr(partner.name)`. `Backpex.Adapters.Ecto` joins a subquery of the values by item, so an aggregate doesn't
-duplicate items. The index view sorts by aggregates and searches them like any other computed value.
+duplicate items. Aggregates that only differ in their `:value`, e.g. the name and the email of the primary partner,
+share one subquery. The index view sorts by aggregates and searches them like any other computed value.
 
 ## Values of the database
 

guides/upgrading/v0.24.md

@@ -381,8 +381,8 @@ The `schema/2` callback of `Backpex.Adapters.Ecto.Field` is gone, as there are n
 e.g. `:backpex_author`, instead of a binding named after its schema.
 
 The new `:options_filter` option of `Backpex.Fields.BelongsTo`, `Backpex.Fields.HasMany` and
-`Backpex.Fields.HasManyThrough` limits the options of the field with an expression, next to the `:options_query` of
-the Ecto adapter.
+`Backpex.Fields.HasManyThrough` limits the options of the field with an expression, and `:options_sort` sorts them,
+e.g. `[name: :asc]`, next to the `:options_query` of the Ecto adapter.
 
 ### Custom fields
 
@@ -403,7 +403,8 @@ The options and callbacks of fields that only `Backpex.Adapters.Ecto` used moved
 
 If your field calls one of these functions on a built-in field, e.g. `search_condition/3` of `Backpex.Fields.Number`,
 build the condition in your field instead. Code that calls `schema/2` on the module of a field, e.g. to build the
-query of an export, raises now. Build the query of the index view with `Backpex.LiveResource.index_query/1` instead:
+query of an export, raises now. Build the query of the index view with `Backpex.LiveResource.index_query/1` instead. It
+takes the fields of the index view of the LiveResource, so it also works with the assigns of a resource action:
 
 ```elixir
 {:ok, items} =

integration_test/adapters/ecto_expressions_test.exs

@@ -115,6 +115,28 @@ defmodule Backpex.Integration.Adapters.EctoExpressionsTest do
     end
   end
 
+  describe "values" do
+    test "compare the values of arithmetic, datetime_diff and to_string" do
+      assert titles(expr(likes * 2 - 10 > 100)) == ["100% Elixir"]
+      assert titles(expr(likes / 2 == 2)) == ["Rust_Notes"]
+      assert titles(expr(to_string(likes) == "50")) == ["No author"]
+      assert titles(expr(datetime_diff(updated_at, inserted_at, :minute) == 0)) |> length() == 3
+      assert titles(expr(concat([title, user.username], " by ") == "Rust_Notes by bob")) == ["Rust_Notes"]
+    end
+
+    test "are the aggregates of metrics" do
+      {:ok, %{likes: likes, minutes: minutes}} =
+        PostLive
+        |> Backpex.Query.new()
+        |> Backpex.Query.aggregate(:likes, {:sum, expr(likes * 2)}, where: expr(likes < 100))
+        |> Backpex.Query.aggregate(:minutes, {:max, expr(datetime_diff(updated_at, inserted_at, :minute))})
+        |> Backpex.Resource.aggregate()
+
+      assert likes == 110
+      assert minutes === 0
+    end
+  end
+
   describe "native expressions" do
     test "are placed as they are" do
       native = Backpex.Expr.native(EctoAdapter, dynamic([post], fragment("lower(?)", post.title) == "rust_notes"))

integration_test/index/metrics_test.exs

@@ -13,4 +13,16 @@ defmodule Backpex.Integration.Index.MetricsTest do
     |> assert_has(".card-body", text: "Total likes 1600 likes", exact: true)
     |> assert_has(".card-body", text: "Published Posts 2", exact: true)
   end
+
+  test "ignores the filters of :ignore_filters", %{conn: conn} do
+    insert(:post, likes: 250, published: true)
+    insert(:post, likes: 1000, published: false)
+
+    # The published filter is active by default.
+    conn
+    |> visit(~p"/admin/posts")
+    |> assert_has("table tbody tr", count: 1)
+    |> assert_has(".card-body", text: "Total likes 250 likes", exact: true)
+    |> assert_has(".card-body", text: "All posts 2", exact: true)
+  end
 end

integration_test/options/computed_test.exs

@@ -26,6 +26,21 @@ defmodule Backpex.Integration.Options.ComputedTest do
     |> assert_has("tbody tr:nth-child(2) td", text: "0", exact: true)
   end
 
+  test "reads aggregates that share a join and computes with aggregates", %{ann: ann, bob: bob} do
+    {:ok, users} =
+      []
+      |> Backpex.Resource.items_query(%{}, Backpex.Test.UserStatsLive)
+      |> Backpex.Query.sort(username: :asc)
+      |> Backpex.Resource.read()
+
+    assert [
+             %{id: ann_id, top_post: "Big", top_post_likes: 9, draft_count: 1},
+             %{id: bob_id, top_post: nil, top_post_likes: nil, draft_count: 0}
+           ] = users
+
+    assert {ann_id, bob_id} == {ann.id, bob.id}
+  end
+
   test "sorts by an aggregate and searches it", %{conn: conn} do
     conn
     |> visit(~p"/admin/user-stats?#{%{order_by: "post_count", order_direction: "asc"}}")

integration_test/options/scope_test.exs

@@ -1,10 +1,12 @@
 defmodule Backpex.Integration.Options.ScopeTest do
   @moduledoc """
-  Checks `c:Backpex.LiveResource.scope/1` and the `:options_filter` option of relation fields through
+  Checks `c:Backpex.LiveResource.scope/1` and the `:options_filter` and `:options_sort` options of relation fields through
   `Backpex.Test.ScopedPostLive`, which shows the published posts only.
   """
   use Backpex.ConnCase, async: true
 
+  import Phoenix.LiveViewTest, only: [live: 2]
+
   describe "scope/1" do
     setup do
       %{
@@ -50,6 +52,22 @@ defmodule Backpex.Integration.Options.ScopeTest do
     end
   end
 
+  describe "options_sort" do
+    test "sorts the options of a belongs to and a has many field", %{conn: conn} do
+      insert(:user, username: "anna")
+      insert(:user, username: "zoe")
+      insert(:tag, name: "Ash")
+      insert(:tag, name: "Zig")
+
+      {:ok, _view, html} = live(conn, ~p"/admin/scoped-posts/new")
+
+      assert before?(html, "zoe", "anna")
+      assert before?(html, "Zig", "Ash")
+    end
+
+    defp before?(html, first, second), do: :binary.match(html, first) < :binary.match(html, second)
+  end
+
   describe "options_filter" do
     test "limits the options of a belongs to field to the items that match the expression", %{conn: conn} do
       insert(:user, username: "visible_author")

integration_test/support/app/live/post_live.ex

@@ -127,6 +127,13 @@ defmodule Backpex.Test.PostLive do
         label: "Published Posts",
         select: dynamic([p], count(fragment("CASE WHEN ? = TRUE THEN 1 ELSE NULL END", p.published))),
         format: &Integer.to_string/1
+      },
+      all_posts: %{
+        module: Value,
+        label: "All posts",
+        aggregate: :count,
+        ignore_filters: [:published],
+        format: &Integer.to_string/1
       }
     ]
   end

integration_test/support/app/live/scoped_post_live.ex

@@ -2,7 +2,7 @@ defmodule Backpex.Test.ScopedPostLive do
   @moduledoc """
   A LiveResource of the published posts only, with `c:Backpex.LiveResource.scope/1`. Its relation fields limit their
   options with `:options_filter`: the author to users whose username doesn't start with `"hidden"`, and the tags of the
-  new form to the ones whose name doesn't start with `"Hidden"`.
+  new form to the ones whose name doesn't start with `"Hidden"`. `:options_sort` sorts both in descending order.
   """
   use Backpex.LiveResource,
     adapter_config: [
@@ -35,7 +35,8 @@ defmodule Backpex.Test.ScopedPostLive do
         label: "Author",
         display_field: :username,
         live_resource: Backpex.Test.UserLive,
-        options_filter: expr(not starts_with(username, "hidden"))
+        options_filter: expr(not starts_with(username, "hidden")),
+        options_sort: [username: :desc]
       },
       tags: %{
         module: Backpex.Fields.HasMany,
@@ -44,7 +45,8 @@ defmodule Backpex.Test.ScopedPostLive do
         live_resource: Backpex.Test.TagLive,
         options_filter: fn assigns ->
           if assigns.live_action == :new, do: expr(not starts_with(name, "Hidden"))
-        end
+        end,
+        options_sort: fn _assigns -> [name: :desc] end
       }
     ]
   end

integration_test/support/app/live/user_stats_live.ex

@@ -1,7 +1,7 @@
 defmodule Backpex.Test.UserStatsLive do
   @moduledoc """
   Users with computed values of their posts: aggregates over a relationship to many items, with a `:where`, a `:value`
-  and a `:sort`, and over the path `posts.tags`.
+  and a `:sort`, over the path `posts.tags`, aggregates that share a join and arithmetic of aggregates.
   """
   use Backpex.LiveResource,
     adapter_config: [
@@ -30,6 +30,9 @@ defmodule Backpex.Test.UserStatsLive do
       published_count: aggregate(:count, :posts, where: expr(published == true)),
       total_likes: aggregate(:sum, :posts, value: expr(likes)),
       top_post: aggregate(:first, :posts, value: expr(title), sort: [likes: :desc]),
+      # Shares the join of `top_post`, as it only differs in the value.
+      top_post_likes: aggregate(:first, :posts, value: expr(likes), sort: [likes: :desc]),
+      draft_count: expr(^aggregate(:count, :posts) - ^aggregate(:count, :posts, where: expr(published == true))),
       first_tag: aggregate(:first, [:posts, :tags], value: expr(name), sort: [name: :asc])
     ]
   end
@@ -42,6 +45,8 @@ defmodule Backpex.Test.UserStatsLive do
       published_count: %{module: Backpex.Fields.Number, label: "Published"},
       total_likes: %{module: Backpex.Fields.Number, label: "Likes"},
       top_post: %{module: Backpex.Fields.Text, label: "Top post", searchable: true},
+      top_post_likes: %{module: Backpex.Fields.Number, label: "Top post likes"},
+      draft_count: %{module: Backpex.Fields.Number, label: "Drafts"},
       first_tag: %{module: Backpex.Fields.Text, label: "First tag"}
     ]
   end

integration_test/support/app/schemas/user.ex

@@ -18,6 +18,8 @@ defmodule Backpex.Test.User do
     field :published_count, :integer, virtual: true
     field :total_likes, :integer, virtual: true
     field :top_post, :string, virtual: true
+    field :top_post_likes, :integer, virtual: true
+    field :draft_count, :integer, virtual: true
     field :first_tag, :string, virtual: true
     field :age, :integer
     field :role, Ecto.Enum, values: [:user, :admin]

lib/backpex/adapters/ecto.ex

@@ -89,6 +89,8 @@ defmodule Backpex.Adapters.Ecto do
   import Ecto.Query
 
   alias Backpex.Adapters.Ecto.Where
+  alias Backpex.Expr.Aggregate
+  alias Backpex.Expr.Call
   alias Backpex.Expr.Native
   alias Backpex.Expr.Ref
   alias Backpex.Query
@@ -189,26 +191,39 @@ defmodule Backpex.Adapters.Ecto do
 
     {:ok,
      Map.new(aggregates, fn %{name: name, aggregate: aggregate, where: where} ->
+       {ecto_query, value} = aggregate_dynamic(base, aggregate)
+
        data =
-         base
+         ecto_query
          |> Where.where_expr(where)
-         |> select(^aggregate_dynamic(aggregate))
+         |> select(^value)
          |> repo.one()
 
        {name, data}
      end)}
   end
 
-  # An aggregate, see `t:Backpex.Metric.aggregate/0`.
-  defp aggregate_dynamic(%Native{adapter: __MODULE__, expr: expression}), do: expression
-  defp aggregate_dynamic(:count), do: dynamic(count())
-  defp aggregate_dynamic({:count, field}) when is_atom(field), do: dynamic([item], count(field(item, ^field)))
-  defp aggregate_dynamic({:sum, field}) when is_atom(field), do: dynamic([item], sum(field(item, ^field)))
-  defp aggregate_dynamic({:avg, field}) when is_atom(field), do: dynamic([item], avg(field(item, ^field)))
-  defp aggregate_dynamic({:min, field}) when is_atom(field), do: dynamic([item], min(field(item, ^field)))
-  defp aggregate_dynamic({:max, field}) when is_atom(field), do: dynamic([item], max(field(item, ^field)))
+  # An aggregate, see `t:Backpex.Metric.aggregate/0`, with the query and the joins its value needs.
+  defp aggregate_dynamic(ecto_query, %Native{adapter: __MODULE__, expr: expression}), do: {ecto_query, expression}
+  defp aggregate_dynamic(ecto_query, :count), do: {ecto_query, dynamic(count())}
+
+  defp aggregate_dynamic(ecto_query, {kind, field}) when is_atom(field),
+    do: aggregate_dynamic(ecto_query, {kind, Backpex.Expr.ref(field)})
+
+  defp aggregate_dynamic(ecto_query, {kind, %struct{} = expression})
+       when kind in [:count, :sum, :avg, :min, :max] and struct in [Ref, Call, Aggregate] do
+    {ecto_query, value} = Where.value(ecto_query, expression)
+
+    case kind do
+      :count -> {ecto_query, dynamic(count(^value))}
+      :sum -> {ecto_query, dynamic(sum(^value))}
+      :avg -> {ecto_query, dynamic(avg(^value))}
+      :min -> {ecto_query, dynamic(min(^value))}
+      :max -> {ecto_query, dynamic(max(^value))}
+    end
+  end
 
-  defp aggregate_dynamic(aggregate) do
+  defp aggregate_dynamic(_ecto_query, aggregate) do
     raise ArgumentError, "expected an aggregate such as :count or {:sum, :field}, got: #{inspect(aggregate)}"
   end
 
@@ -514,7 +529,9 @@ defmodule Backpex.Adapters.Ecto do
 
   defp merge_computed(ecto_query, %Query{computed: computed}) when computed == %{}, do: ecto_query
 
-  defp merge_computed(ecto_query, %Query{computed: computed}) do
+  defp merge_computed(ecto_query, %Query{computed: computed} = query) do
+    ecto_query = Where.join_aggregates(ecto_query, aggregate_expressions(query))
+
     {values, ecto_query} =
       Enum.map_reduce(computed, ecto_query, fn {name, expression}, ecto_query ->
         {ecto_query, value} = Where.value(ecto_query, expression)
@@ -524,6 +541,16 @@ defmodule Backpex.Adapters.Ecto do
     select_merge(ecto_query, ^Map.new(values))
   end
 
+  # The expressions whose aggregates share joins, see `Backpex.Adapters.Ecto.Where.join_aggregates/2`: the computed
+  # values, and the ones of related items that sorts and the search refer to.
+  defp aggregate_expressions(%Query{computed: computed, search: search} = query) do
+    sort_refs = query |> Query.sorts() |> Enum.map(fn {ref, _direction} -> ref end)
+    search_refs = if search, do: search.targets, else: []
+    related = Enum.flat_map(sort_refs ++ search_refs, fn ref -> query |> Query.computed_expr(ref) |> List.wrap() end)
+
+    Map.values(computed) ++ related
+  end
+
   # --- QUERIES
 
   @doc """