APIv2 - initial PR (plausible#4216)

* WIP new querying * WIP: Move some aggregate code under new command * WIP: Add joins, handling less metrics * join events table to sessions if needed * Merge imported results with built query * Remove dead code * WIP: /api/v2/query * Allow grouping by time * Use JOIN for main query * Build query result * update parse_time * Make joinless order by work * First test * more breakdown tests * Serialize event:goal filters in an json-encodable way/reflection * Handle inner vs outer ORDER BY clauses properly * Handle single conversion_rate metric * Update more tests * Get parsing tests passing again * Validate filtered goal filter is configured * Enable more validation tests * Enable more event:name breakdown tests * Enable more breakdown tests * Validate site has access to custom props * Validate conversion_rate metric which is only allowed in some situations * Validate that empty event:props: is not valid * handle query.dimensions properly in table_decider * test more validations on metrics/dimensions * Validate session metrics in combination with event dimension(s) * Tests cleanup * Parse include.imports * Get imports working with new querying * Make more imports tests work * Make event:props:path imports-adjacent test work * Get query imports warning-related tests running * Remove dead pagination tests * Solve dead import * Solve some warnings * Update aggregate metrics tests * credo * Improve test naming * Lazy goal loading * Use datetime methods * Ecto -> SQL module name * Remove Expression.dimension mode option
yonasBSD · Jun 25, 2024 · 58a66a9 · 58a66a9
1 parent 5382020
commit 58a66a9
Show file tree

Hide file tree

Showing 20 changed files with 4,228 additions and 259 deletions.
diff --git a/extra/lib/plausible/stats/goal/revenue.ex b/extra/lib/plausible/stats/goal/revenue.ex
@@ -15,14 +15,20 @@ defmodule Plausible.Stats.Goal.Revenue do
   def total_revenue_query() do
     dynamic(
       [e],
-      fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)
+      selected_as(
+        fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount),
+        :total_revenue
+      )
     )
   end
 
   def average_revenue_query() do
     dynamic(
       [e],
-      fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)
+      selected_as(
+        fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount),
+        :average_revenue
+      )
     )
   end
 

diff --git a/lib/plausible/stats.ex b/lib/plausible/stats.ex
@@ -1,12 +1,16 @@
 defmodule Plausible.Stats do
   use Plausible
+  alias Plausible.Stats.QueryResult
+  use Plausible.ClickhouseRepo
 
   alias Plausible.Stats.{
     Breakdown,
     Aggregate,
     Timeseries,
     CurrentVisitors,
-    FilterSuggestions
+    FilterSuggestions,
+    QueryOptimizer,
+    SQL
   }
 
   use Plausible.DebugReplayInfo
@@ -31,6 +35,15 @@ defmodule Plausible.Stats do
     CurrentVisitors.current_visitors(site)
   end
 
+  def query(site, query) do
+    optimized_query = QueryOptimizer.optimize(query)
+
+    optimized_query
+    |> SQL.QueryBuilder.build(site)
+    |> ClickhouseRepo.all()
+    |> QueryResult.from(optimized_query)
+  end
+
   on_ee do
     def funnel(site, query, funnel) do
       include_sentry_replay_info()

diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex
@@ -1,7 +1,7 @@
 defmodule Plausible.Stats.Aggregate do
   use Plausible.ClickhouseRepo
   use Plausible
-  import Plausible.Stats.{Base, Imported}
+  import Plausible.Stats.Base
   import Ecto.Query
   alias Plausible.Stats.{Query, Util}
 
@@ -15,23 +15,24 @@ defmodule Plausible.Stats.Aggregate do
 
     Query.trace(query, metrics)
 
-    {event_metrics, session_metrics, other_metrics} =
-      metrics
-      |> Util.maybe_add_visitors_metric()
-      |> Plausible.Stats.TableDecider.partition_metrics(query)
+    query_with_metrics = %Plausible.Stats.Query{
+      query
+      | metrics: Util.maybe_add_visitors_metric(metrics)
+    }
 
-    event_task = fn -> aggregate_events(site, query, event_metrics) end
-
-    session_task = fn -> aggregate_sessions(site, query, session_metrics) end
+    q = Plausible.Stats.SQL.QueryBuilder.build(query_with_metrics, site)
 
     time_on_page_task =
-      if :time_on_page in other_metrics do
+      if :time_on_page in query_with_metrics.metrics do
         fn -> aggregate_time_on_page(site, query) end
       else
         fn -> %{} end
       end
 
-    Plausible.ClickhouseRepo.parallel_tasks([session_task, event_task, time_on_page_task])
+    Plausible.ClickhouseRepo.parallel_tasks([
+      run_query_task(q),
+      time_on_page_task
+    ])
     |> Enum.reduce(%{}, fn aggregate, task_result -> Map.merge(aggregate, task_result) end)
     |> Util.keep_requested_metrics(metrics)
     |> cast_revenue_metrics_to_money(currency)
@@ -40,24 +41,8 @@ defmodule Plausible.Stats.Aggregate do
     |> Enum.into(%{})
   end
 
-  defp aggregate_events(_, _, []), do: %{}
-
-  defp aggregate_events(site, query, metrics) do
-    from(e in base_event_query(site, query), select: ^select_event_metrics(metrics))
-    |> merge_imported(site, query, metrics)
-    |> maybe_add_conversion_rate(site, query, metrics)
-    |> ClickhouseRepo.one()
-  end
-
-  defp aggregate_sessions(_, _, []), do: %{}
-
-  defp aggregate_sessions(site, query, metrics) do
-    from(e in query_sessions(site, query), select: ^select_session_metrics(metrics, query))
-    |> filter_converted_sessions(site, query)
-    |> merge_imported(site, query, metrics)
-    |> ClickhouseRepo.one()
-    |> Util.keep_requested_metrics(metrics)
-  end
+  defp run_query_task(nil), do: fn -> %{} end
+  defp run_query_task(q), do: fn -> ClickhouseRepo.one(q) end
 
   defp aggregate_time_on_page(site, query) do
     windowed_pages_q =

diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex
@@ -31,7 +31,7 @@ defmodule Plausible.Stats.Base do
     end
   end
 
-  defp query_events(site, query) do
+  def query_events(site, query) do
     q = from(e in "events_v2", where: ^Filters.WhereBuilder.build(:events, site, query))
 
     on_ee do
@@ -62,14 +62,21 @@ defmodule Plausible.Stats.Base do
       pageviews:
         dynamic(
           [e],
-          fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name)
+          selected_as(
+            fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name),
+            :pageviews
+          )
         )
     }
   end
 
   defp select_event_metric(:events) do
     %{
-      events: dynamic([], fragment("toUInt64(round(count(*) * any(_sample_factor)))"))
+      events:
+        dynamic(
+          [],
+          selected_as(fragment("toUInt64(round(count(*) * any(_sample_factor)))"), :events)
+        )
     }
   end
 
@@ -82,7 +89,13 @@ defmodule Plausible.Stats.Base do
   defp select_event_metric(:visits) do
     %{
       visits:
-        dynamic([e], fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.session_id))
+        dynamic(
+          [e],
+          selected_as(
+            fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.session_id),
+            :visits
+          )
+        )
     }
   end
 
@@ -127,10 +140,13 @@ defmodule Plausible.Stats.Base do
       bounce_rate:
         dynamic(
           [],
-          fragment(
-            "toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))",
-            ^condition,
-            ^condition
+          selected_as(
+            fragment(
+              "toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))",
+              ^condition,
+              ^condition
+            ),
+            :bounce_rate
           )
         ),
       __internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))
@@ -139,7 +155,14 @@ defmodule Plausible.Stats.Base do
 
   defp select_session_metric(:visits, _query) do
     %{
-      visits: dynamic([s], fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign))
+      visits:
+        dynamic(
+          [s],
+          selected_as(
+            fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign),
+            :visits
+          )
+        )
     }
   end
 
@@ -148,7 +171,10 @@ defmodule Plausible.Stats.Base do
       pageviews:
         dynamic(
           [s],
-          fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews)
+          selected_as(
+            fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews),
+            :pageviews
+          )
         )
     }
   end
@@ -158,7 +184,10 @@ defmodule Plausible.Stats.Base do
       events:
         dynamic(
           [s],
-          fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events)
+          selected_as(
+            fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events),
+            :events
+          )
         )
     }
   end
@@ -179,7 +208,13 @@ defmodule Plausible.Stats.Base do
   defp select_session_metric(:visit_duration, _query) do
     %{
       visit_duration:
-        dynamic([], fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))")),
+        dynamic(
+          [],
+          selected_as(
+            fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))"),
+            :visit_duration
+          )
+        ),
       __internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))
     }
   end
@@ -189,7 +224,15 @@ defmodule Plausible.Stats.Base do
       views_per_visit:
         dynamic(
           [s],
-          fragment("ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)", s.sign, s.pageviews, s.sign)
+          selected_as(
+            fragment(
+              "ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)",
+              s.sign,
+              s.pageviews,
+              s.sign
+            ),
+            :views_per_visit
+          )
         ),
       __internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))
     }
@@ -328,11 +371,14 @@ defmodule Plausible.Stats.Base do
       )
       |> select_merge(%{
         percentage:
-          fragment(
-            "if(? > 0, round(? / ? * 100, 1), null)",
-            selected_as(:__total_visitors),
-            selected_as(:visitors),
-            selected_as(:__total_visitors)
+          selected_as(
+            fragment(
+              "if(? > 0, round(? / ? * 100, 1), null)",
+              selected_as(:__total_visitors),
+              selected_as(:visitors),
+              selected_as(:__total_visitors)
+            ),
+            :percentage
           )
       })
     else
@@ -357,11 +403,14 @@ defmodule Plausible.Stats.Base do
       )
       |> select_merge([e], %{
         conversion_rate:
-          fragment(
-            "if(? > 0, round(? / ? * 100, 1), 0)",
-            selected_as(:__total_visitors),
-            e.visitors,
-            selected_as(:__total_visitors)
+          selected_as(
+            fragment(
+              "if(? > 0, round(? / ? * 100, 1), 0)",
+              selected_as(:__total_visitors),
+              e.visitors,
+              selected_as(:__total_visitors)
+            ),
+            :conversion_rate
           )
       })
     else

diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex
@@ -276,8 +276,8 @@ defmodule Plausible.Stats.Breakdown do
   defp breakdown_table(%Query{dimensions: ["visit:exit_page"]}, _metrics), do: :session
   defp breakdown_table(%Query{dimensions: ["visit:exit_page_hostname"]}, _metrics), do: :session
 
-  defp breakdown_table(%Query{dimensions: [dimension]} = query, metrics) do
-    {_, session_metrics, _} = TableDecider.partition_metrics(metrics, query, dimension)
+  defp breakdown_table(%Query{dimensions: [_dimension]} = query, metrics) do
+    {_, session_metrics, _} = TableDecider.partition_metrics(metrics, query)
 
     if not Enum.empty?(session_metrics) do
       :session

diff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.ex
@@ -87,6 +87,6 @@ defmodule Plausible.Stats.Filters do
     property
     |> String.split(":")
     |> List.last()
-    |> String.to_existing_atom()
+    |> String.to_atom()
   end
 end