From 2eeaf7a1528728af0d39e5465b2732a196eda3fc Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Fri, 28 Jun 2024 08:59:54 +0300 Subject: [PATCH] APIv2: Aggregates, timeseries, conversion_rate, hostname (#4251) * Add some aggregates tests * Port aggregates tests to do with filtering * Session metrics can be queried with event: filters * Solve a typo * Update a validation message * Add validations for views_per_visit * Port an aggregation/imports test * Optimize time dimension, add tests * Add first timeseries test, update parsing tests * Docs for SQL.Expression * Test timeseries more * Allow time explicitly in order_by * Add multiple breakdowns test * Refactor QueryOptimizer not to care about time dimension placement in dimensions array * Add test breaking down by event:hostname * Add hostname filtering logic to QueryOptimizer, unblock some tests * WIP: Breakdown by goal * conversion rate logic for query api * Update more tests * Set default order_by * dimension_label * preloaded_goals in tests * inline load_goals * Use Date functions over Timex * Comments * is_binary * Remove special form used in tests * Fix defmodule * WIP: Fix memory leak, event:page breakdown logic * Enable more tests, fix for group_conversion_rate without explicit visitors metric * Re-enable a partially commented test * Re-enable a partially commented test * Get last test passing * No imports order_by in apiv2 * Add a TODO * Remove redundant Util call * Update aggregate.ex * Remove problematic test --- lib/plausible/stats/aggregate.ex | 5 +- lib/plausible/stats/base.ex | 7 +- lib/plausible/stats/filters/filters.ex | 2 +- lib/plausible/stats/filters/query_parser.ex | 140 +- lib/plausible/stats/filters/utils.ex | 14 + lib/plausible/stats/imported/imported.ex | 144 +- lib/plausible/stats/metrics.ex | 1 + lib/plausible/stats/query.ex | 17 +- lib/plausible/stats/query_optimizer.ex | 139 +- lib/plausible/stats/query_result.ex | 20 +- lib/plausible/stats/sql/expression.ex | 22 +- lib/plausible/stats/sql/query_builder.ex | 199 +- lib/plausible/stats/table_decider.ex | 7 +- lib/plausible/stats/util.ex | 11 +- test/plausible/stats/query_optimizer_test.exs | 147 + test/plausible/stats/query_parser_test.exs | 187 +- .../external_stats_controller/query_test.exs | 3246 ++++++++++------- 17 files changed, 2906 insertions(+), 1402 deletions(-) create mode 100644 test/plausible/stats/query_optimizer_test.exs diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex index 82d1c8a180e8..9a2e65bab607 100644 --- a/lib/plausible/stats/aggregate.ex +++ b/lib/plausible/stats/aggregate.ex @@ -15,10 +15,7 @@ defmodule Plausible.Stats.Aggregate do Query.trace(query, metrics) - query_with_metrics = %Plausible.Stats.Query{ - query - | metrics: Util.maybe_add_visitors_metric(metrics) - } + query_with_metrics = %Query{query | metrics: metrics} q = Plausible.Stats.SQL.QueryBuilder.build(query_with_metrics, site) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index bd01257fb7eb..3b3dfd2e338d 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -121,6 +121,7 @@ defmodule Plausible.Stats.Base do defp select_event_metric(:percentage), do: %{} defp select_event_metric(:conversion_rate), do: %{} + defp select_event_metric(:group_conversion_rate), do: %{} defp select_event_metric(:total_visitors), do: %{} defp select_event_metric(unknown), do: raise("Unknown metric: #{unknown}") @@ -344,9 +345,9 @@ defmodule Plausible.Stats.Base do # only if it's included in the base query - otherwise the total will be based on # a different data set, making the metric inaccurate. This is why we're using an # explicit `include_imported` argument here. - defp total_visitors_subquery(site, query, include_imported) + def total_visitors_subquery(site, query, include_imported) - defp total_visitors_subquery(site, query, true = _include_imported) do + def total_visitors_subquery(site, query, true = _include_imported) do dynamic( [e], selected_as( @@ -357,7 +358,7 @@ defmodule Plausible.Stats.Base do ) end - defp total_visitors_subquery(site, query, false = _include_imported) do + def total_visitors_subquery(site, query, false = _include_imported) do dynamic([e], selected_as(subquery(total_visitors(site, query)), :__total_visitors)) end diff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.ex index dc13c7dcd9cc..b6760965edf7 100644 --- a/lib/plausible/stats/filters/filters.ex +++ b/lib/plausible/stats/filters/filters.ex @@ -87,6 +87,6 @@ defmodule Plausible.Stats.Filters do property |> String.split(":") |> List.last() - |> String.to_atom() + |> String.to_existing_atom() end end diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.ex index 50232618995e..0184478905b2 100644 --- a/lib/plausible/stats/filters/query_parser.ex +++ b/lib/plausible/stats/filters/query_parser.ex @@ -5,13 +5,15 @@ defmodule Plausible.Stats.Filters.QueryParser do alias Plausible.Stats.Filters alias Plausible.Stats.Query - def parse(site, params) when is_map(params) do + def parse(site, params, now \\ nil) when is_map(params) do with {:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])), {:ok, filters} <- parse_filters(Map.get(params, "filters", [])), - {:ok, date_range} <- parse_date_range(site, Map.get(params, "date_range")), + {:ok, date_range} <- + parse_date_range(site, Map.get(params, "date_range"), now || today(site)), {:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])), {:ok, order_by} <- parse_order_by(Map.get(params, "order_by")), {:ok, include} <- parse_include(Map.get(params, "include", %{})), + preloaded_goals <- preload_goals_if_needed(site, filters, dimensions), query = %{ metrics: metrics, filters: filters, @@ -19,10 +21,11 @@ defmodule Plausible.Stats.Filters.QueryParser do dimensions: dimensions, order_by: order_by, timezone: site.timezone, - imported_data_requested: Map.get(include, :imports, false) + imported_data_requested: Map.get(include, :imports, false), + preloaded_goals: preloaded_goals }, :ok <- validate_order_by(query), - :ok <- validate_goal_filters(site, query), + :ok <- validate_goal_filters(query), :ok <- validate_custom_props_access(site, query), :ok <- validate_metrics(query) do {:ok, query} @@ -43,12 +46,14 @@ defmodule Plausible.Stats.Filters.QueryParser do defp parse_metric("time_on_page"), do: {:ok, :time_on_page} defp parse_metric("conversion_rate"), do: {:ok, :conversion_rate} + defp parse_metric("group_conversion_rate"), do: {:ok, :group_conversion_rate} defp parse_metric("visitors"), do: {:ok, :visitors} defp parse_metric("pageviews"), do: {:ok, :pageviews} defp parse_metric("events"), do: {:ok, :events} defp parse_metric("visits"), do: {:ok, :visits} defp parse_metric("bounce_rate"), do: {:ok, :bounce_rate} defp parse_metric("visit_duration"), do: {:ok, :visit_duration} + defp parse_metric("views_per_visit"), do: {:ok, :views_per_visit} defp parse_metric(unknown_metric), do: {:error, "Unknown metric '#{inspect(unknown_metric)}'"} def parse_filters(filters) when is_list(filters) do @@ -84,7 +89,7 @@ defmodule Plausible.Stats.Filters.QueryParser do do: parse_clauses_list(filter) defp parse_clauses_list([_operation, filter_key, list] = filter) when is_list(list) do - all_strings? = Enum.all?(list, &is_bitstring/1) + all_strings? = Enum.all?(list, &is_binary/1) cond do filter_key == "event:goal" && all_strings? -> {:ok, [Filters.Utils.wrap_goal_value(list)]} @@ -95,27 +100,62 @@ defmodule Plausible.Stats.Filters.QueryParser do defp parse_clauses_list(filter), do: {:error, "Invalid filter '#{inspect(filter)}'"} - defp parse_date_range(site, "day") do - today = DateTime.now!(site.timezone) |> DateTime.to_date() - {:ok, Date.range(today, today)} + defp parse_date_range(_site, "day", date) do + {:ok, Date.range(date, date)} end - defp parse_date_range(_site, "7d"), do: {:ok, "7d"} - defp parse_date_range(_site, "30d"), do: {:ok, "30d"} - defp parse_date_range(_site, "month"), do: {:ok, "month"} - defp parse_date_range(_site, "6mo"), do: {:ok, "6mo"} - defp parse_date_range(_site, "12mo"), do: {:ok, "12mo"} - defp parse_date_range(_site, "year"), do: {:ok, "year"} + defp parse_date_range(_site, "7d", last) do + first = last |> Date.add(-6) + {:ok, Date.range(first, last)} + end + + defp parse_date_range(_site, "30d", last) do + first = last |> Date.add(-30) + {:ok, Date.range(first, last)} + end + + defp parse_date_range(_site, "month", today) do + last = today |> Date.end_of_month() + first = last |> Date.beginning_of_month() + {:ok, Date.range(first, last)} + end + + defp parse_date_range(_site, "6mo", today) do + last = today |> Date.end_of_month() + + first = + last + |> Timex.shift(months: -5) + |> Date.beginning_of_month() + + {:ok, Date.range(first, last)} + end - defp parse_date_range(site, "all") do - today = DateTime.now!(site.timezone) |> DateTime.to_date() + defp parse_date_range(_site, "12mo", today) do + last = today |> Date.end_of_month() + + first = + last + |> Timex.shift(months: -11) + |> Date.beginning_of_month() + + {:ok, Date.range(first, last)} + end + + defp parse_date_range(_site, "year", today) do + last = today |> Timex.end_of_year() + first = last |> Timex.beginning_of_year() + {:ok, Date.range(first, last)} + end + + defp parse_date_range(site, "all", today) do start_date = Plausible.Sites.stats_start_date(site) || today {:ok, Date.range(start_date, today)} end - defp parse_date_range(_site, [from_date_string, to_date_string]) - when is_bitstring(from_date_string) and is_bitstring(to_date_string) do + defp parse_date_range(_site, [from_date_string, to_date_string], _date) + when is_binary(from_date_string) and is_binary(to_date_string) do with {:ok, from_date} <- Date.from_iso8601(from_date_string), {:ok, to_date} <- Date.from_iso8601(to_date_string) do {:ok, Date.range(from_date, to_date)} @@ -124,7 +164,10 @@ defmodule Plausible.Stats.Filters.QueryParser do end end - defp parse_date_range(_site, unknown), do: {:error, "Invalid date range '#{inspect(unknown)}'"} + defp parse_date_range(_site, unknown, _), + do: {:error, "Invalid date_range '#{inspect(unknown)}'"} + + defp today(site), do: DateTime.now!(site.timezone) |> DateTime.to_date() defp parse_dimensions(dimensions) when is_list(dimensions) do if length(dimensions) == length(Enum.uniq(dimensions)) do @@ -178,10 +221,9 @@ defmodule Plausible.Stats.Filters.QueryParser do end defp parse_time("time"), do: {:ok, "time"} + defp parse_time("time:hour"), do: {:ok, "time:hour"} defp parse_time("time:day"), do: {:ok, "time:day"} - defp parse_time("time:week"), do: {:ok, "time:week"} defp parse_time("time:month"), do: {:ok, "time:month"} - defp parse_time("time:year"), do: {:ok, "time:year"} defp parse_time(_), do: :error defp parse_order_direction([_, "asc"]), do: {:ok, :asc} @@ -242,7 +284,22 @@ defmodule Plausible.Stats.Filters.QueryParser do end end - defp validate_goal_filters(site, query) do + defp preload_goals_if_needed(site, filters, dimensions) do + goal_filters? = + Enum.any?(filters, fn [_, filter_key | _rest] -> filter_key == "event:goal" end) + + if goal_filters? or Enum.member?(dimensions, "event:goal") do + Plausible.Goals.for_site(site) + |> Enum.map(fn + %{page_path: path} when is_binary(path) -> {:page, path} + %{event_name: event_name} -> {:event, event_name} + end) + else + [] + end + end + + defp validate_goal_filters(query) do goal_filter_clauses = Enum.flat_map(query.filters, fn [_operation, "event:goal", clauses] -> clauses @@ -250,14 +307,7 @@ defmodule Plausible.Stats.Filters.QueryParser do end) if length(goal_filter_clauses) > 0 do - configured_goals = - Plausible.Goals.for_site(site) - |> Enum.map(fn - %{page_path: path} when is_binary(path) -> {:page, path} - %{event_name: event_name} -> {:event, event_name} - end) - - validate_list(goal_filter_clauses, &validate_goal_filter(&1, configured_goals)) + validate_list(goal_filter_clauses, &validate_goal_filter(&1, query.preloaded_goals)) else :ok end @@ -298,14 +348,12 @@ defmodule Plausible.Stats.Filters.QueryParser do end defp validate_metrics(query) do - validate_list(query.metrics, &validate_metric(&1, query)) - with :ok <- validate_list(query.metrics, &validate_metric(&1, query)) do validate_no_metrics_filters_conflict(query) end end - defp validate_metric(:conversion_rate = metric, query) do + defp validate_metric(metric, query) when metric in [:conversion_rate, :group_conversion_rate] do if Enum.member?(query.dimensions, "event:goal") or not is_nil(Query.get_filter(query, "event:goal")) do :ok @@ -314,20 +362,42 @@ defmodule Plausible.Stats.Filters.QueryParser do end end + defp validate_metric(:views_per_visit = metric, query) do + cond do + not is_nil(Query.get_filter(query, "event:page")) -> + {:error, "Metric `#{metric}` cannot be queried with a filter on `event:page`"} + + length(query.dimensions) > 0 -> + {:error, "Metric `#{metric}` cannot be queried with `dimensions`"} + + true -> + :ok + end + end + defp validate_metric(_, _), do: :ok defp validate_no_metrics_filters_conflict(query) do {_event_metrics, sessions_metrics, _other_metrics} = TableDecider.partition_metrics(query.metrics, query) - if Enum.empty?(sessions_metrics) or not TableDecider.event_filters?(query) do + if Enum.empty?(sessions_metrics) or + not event_dimensions_not_allowing_session_metrics?(query.dimensions) do :ok else {:error, - "Session metric(s) `#{sessions_metrics |> Enum.join(", ")}` cannot be queried along with event filters or dimensions"} + "Session metric(s) `#{sessions_metrics |> Enum.join(", ")}` cannot be queried along with event dimensions"} end end + def event_dimensions_not_allowing_session_metrics?(dimensions) do + Enum.any?(dimensions, fn + "event:page" -> false + "event:" <> _ -> true + _ -> false + end) + end + defp parse_list(list, parser_function) do Enum.reduce_while(list, {:ok, []}, fn value, {:ok, results} -> case parser_function.(value) do diff --git a/lib/plausible/stats/filters/utils.ex b/lib/plausible/stats/filters/utils.ex index 396b62e13e65..96606c1d02ef 100644 --- a/lib/plausible/stats/filters/utils.ex +++ b/lib/plausible/stats/filters/utils.ex @@ -66,4 +66,18 @@ defmodule Plausible.Stats.Filters.Utils do def unwrap_goal_value(goals) when is_list(goals), do: Enum.map(goals, &unwrap_goal_value/1) def unwrap_goal_value({:page, page}), do: "Visit " <> page def unwrap_goal_value({:event, event}), do: event + + def split_goals(goals) do + Enum.split_with(goals, fn {type, _} -> type == :event end) + end + + def split_goals_query_expressions(goals) do + {event_goals, pageview_goals} = split_goals(goals) + events = Enum.map(event_goals, fn {_, event} -> event end) + + page_regexes = + Enum.map(pageview_goals, fn {_, path} -> Plausible.Stats.Base.page_regex(path) end) + + {events, page_regexes} + end end diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.ex index 5c1d77cc340f..6542ab44f31c 100644 --- a/lib/plausible/stats/imported/imported.ex +++ b/lib/plausible/stats/imported/imported.ex @@ -1,4 +1,5 @@ defmodule Plausible.Stats.Imported do + alias Plausible.Stats.Filters use Plausible.ClickhouseRepo import Ecto.Query @@ -266,6 +267,42 @@ defmodule Plausible.Stats.Imported do def merge_imported(q, _, %Query{include_imported: false}, _), do: q + # Note: Only called for APIv2, old APIs use merge_imported_pageview_goals + def merge_imported(q, site, %Query{dimensions: ["event:goal"]} = query, metrics) + when query.v2 do + {events, page_regexes} = Filters.Utils.split_goals_query_expressions(query.preloaded_goals) + + events_q = + "imported_custom_events" + |> Imported.Base.query_imported(site, query) + |> where([i], i.visitors > 0) + |> select_merge([i], %{ + dim0: selected_as(fragment("-indexOf(?, ?)", ^events, i.name), :dim0) + }) + |> select_imported_metrics(metrics) + |> group_by([], selected_as(:dim0)) + |> where([], selected_as(:dim0) != 0) + + pages_q = + "imported_pages" + |> Imported.Base.query_imported(site, query) + |> where([i], i.visitors > 0) + |> where( + [i], + fragment("notEmpty(multiMatchAllIndices(?, ?) as indices)", i.page, ^page_regexes) + ) + |> join(:array, index in fragment("indices")) + |> group_by([_i, index], index) + |> select_merge([_i, index], %{ + dim0: type(fragment("?", index), :integer) + }) + |> select_imported_metrics(metrics) + + q + |> naive_dimension_join(events_q, metrics) + |> naive_dimension_join(pages_q, metrics) + end + def merge_imported(q, site, %Query{dimensions: [dimension]} = query, metrics) when dimension in @imported_properties do dim = Plausible.Stats.Filters.without_prefix(dimension) @@ -289,7 +326,7 @@ defmodule Plausible.Stats.Imported do dynamic([s, i], s.browser == i.browser and s.browser_version == i.browser_version) dim -> - dynamic([s, i], field(s, ^dim) == field(i, ^dim)) + dynamic([s, i], field(s, ^shortname(query, dim)) == field(i, ^shortname(query, dim))) end from(s in Ecto.Query.subquery(q), @@ -299,7 +336,7 @@ defmodule Plausible.Stats.Imported do ) |> select_joined_dimension(dim, query) |> select_joined_metrics(metrics) - |> apply_order_by(metrics) + |> apply_order_by(query, metrics) end def merge_imported(q, site, %Query{dimensions: []} = query, metrics) do @@ -357,6 +394,10 @@ defmodule Plausible.Stats.Imported do |> select_merge([i], %{total_visitors: fragment("sum(?)", i.visitors)}) end + # :TRICKY: Handle backwards compatibility with old breakdown module + defp shortname(query, _dim) when query.v2, do: :dim0 + defp shortname(_query, dim), do: dim + defp select_imported_metrics(q, []), do: q defp select_imported_metrics(q, [:visitors | rest]) do @@ -551,63 +592,71 @@ defmodule Plausible.Stats.Imported do |> select_imported_metrics(rest) end - defp group_imported_by(q, dim, _query) when dim in [:source, :referrer] do + defp group_imported_by(q, dim, query) when dim in [:source, :referrer] do q |> group_by([i], field(i, ^dim)) |> select_merge([i], %{ - ^dim => fragment("if(empty(?), ?, ?)", field(i, ^dim), @no_ref, field(i, ^dim)) + ^shortname(query, dim) => + fragment( + "if(empty(?), ?, ?)", + field(i, ^dim), + @no_ref, + field(i, ^dim) + ) }) end - defp group_imported_by(q, dim, _query) + defp group_imported_by(q, dim, query) when dim in [:utm_source, :utm_medium, :utm_campaign, :utm_term, :utm_content] do q |> group_by([i], field(i, ^dim)) |> where([i], fragment("not empty(?)", field(i, ^dim))) - |> select_merge([i], %{^dim => field(i, ^dim)}) + |> select_merge([i], %{^shortname(query, dim) => field(i, ^dim)}) end - defp group_imported_by(q, :page, _query) do + defp group_imported_by(q, :page, query) do q |> group_by([i], i.page) - |> select_merge([i], %{page: i.page, time_on_page: sum(i.time_on_page)}) + |> select_merge([i], %{^shortname(query, :page) => i.page, time_on_page: sum(i.time_on_page)}) end - defp group_imported_by(q, :country, _query) do + defp group_imported_by(q, :country, query) do q |> group_by([i], i.country) |> where([i], i.country != "ZZ") - |> select_merge([i], %{country: i.country}) + |> select_merge([i], %{^shortname(query, :country) => i.country}) end - defp group_imported_by(q, :region, _query) do + defp group_imported_by(q, :region, query) do q |> group_by([i], i.region) |> where([i], i.region != "") - |> select_merge([i], %{region: i.region}) + |> select_merge([i], %{^shortname(query, :region) => i.region}) end - defp group_imported_by(q, :city, _query) do + defp group_imported_by(q, :city, query) do q |> group_by([i], i.city) |> where([i], i.city != 0 and not is_nil(i.city)) - |> select_merge([i], %{city: i.city}) + |> select_merge([i], %{^shortname(query, :city) => i.city}) end - defp group_imported_by(q, dim, _query) when dim in [:device, :browser] do + defp group_imported_by(q, dim, query) when dim in [:device, :browser] do q |> group_by([i], field(i, ^dim)) |> select_merge([i], %{ - ^dim => fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim)) + ^shortname(query, dim) => + fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim)) }) end - defp group_imported_by(q, :browser_version, _query) do + defp group_imported_by(q, :browser_version, query) do q |> group_by([i], [i.browser, i.browser_version]) |> select_merge([i], %{ - browser: fragment("if(empty(?), ?, ?)", i.browser, @not_set, i.browser), - browser_version: + ^shortname(query, :browser) => + fragment("if(empty(?), ?, ?)", i.browser, @not_set, i.browser), + ^shortname(query, :browser_version) => fragment( "if(empty(?), ?, ?)", i.browser_version, @@ -617,20 +666,22 @@ defmodule Plausible.Stats.Imported do }) end - defp group_imported_by(q, :os, _query) do + defp group_imported_by(q, :os, query) do q |> group_by([i], i.operating_system) |> select_merge([i], %{ - os: fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system) + ^shortname(query, :os) => + fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system) }) end - defp group_imported_by(q, :os_version, _query) do + defp group_imported_by(q, :os_version, query) do q |> group_by([i], [i.operating_system, i.operating_system_version]) |> select_merge([i], %{ - os: fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system), - os_version: + ^shortname(query, :os) => + fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system), + ^shortname(query, :os_version) => fragment( "if(empty(?), ?, ?)", i.operating_system_version, @@ -640,23 +691,23 @@ defmodule Plausible.Stats.Imported do }) end - defp group_imported_by(q, dim, _query) when dim in [:entry_page, :exit_page] do + defp group_imported_by(q, dim, query) when dim in [:entry_page, :exit_page] do q |> group_by([i], field(i, ^dim)) - |> select_merge([i], %{^dim => field(i, ^dim)}) + |> select_merge([i], %{^shortname(query, dim) => field(i, ^dim)}) end - defp group_imported_by(q, :name, _query) do + defp group_imported_by(q, :name, query) do q |> group_by([i], i.name) - |> select_merge([i], %{name: i.name}) + |> select_merge([i], %{^shortname(query, :name) => i.name}) end defp group_imported_by(q, :url, query) when query.v2 do q |> group_by([i], i.link_url) |> select_merge([i], %{ - url: fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none) + ^shortname(query, :url) => fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none) }) end @@ -672,7 +723,7 @@ defmodule Plausible.Stats.Imported do q |> group_by([i], i.path) |> select_merge([i], %{ - path: fragment("if(not empty(?), ?, ?)", i.path, i.path, @none) + ^shortname(query, :path) => fragment("if(not empty(?), ?, ?)", i.path, i.path, @none) }) end @@ -684,9 +735,9 @@ defmodule Plausible.Stats.Imported do }) end - defp select_joined_dimension(q, :city, _query) do + defp select_joined_dimension(q, :city, query) do select_merge(q, [s, i], %{ - city: fragment("greatest(?,?)", i.city, s.city) + ^shortname(query, :city) => fragment("greatest(?,?)", i.city, s.city) }) end @@ -717,9 +768,15 @@ defmodule Plausible.Stats.Imported do }) end - defp select_joined_dimension(q, dim, _query) do + defp select_joined_dimension(q, dim, query) do select_merge(q, [s, i], %{ - ^dim => fragment("if(empty(?), ?, ?)", field(s, ^dim), field(i, ^dim), field(s, ^dim)) + ^shortname(query, dim) => + fragment( + "if(empty(?), ?, ?)", + field(s, ^shortname(query, dim)), + field(i, ^shortname(query, dim)), + field(s, ^shortname(query, dim)) + ) }) end @@ -825,10 +882,23 @@ defmodule Plausible.Stats.Imported do |> select_joined_metrics(rest) end - defp apply_order_by(q, [:visitors | rest]) do + defp apply_order_by(q, %Query{v2: true}, _), do: q + + defp apply_order_by(q, query, [:visitors | rest]) do order_by(q, [s, i], desc: s.visitors + i.visitors) - |> apply_order_by(rest) + |> apply_order_by(query, rest) end - defp apply_order_by(q, _), do: q + defp apply_order_by(q, _query, _), do: q + + defp naive_dimension_join(q1, q2, metrics) do + from(a in Ecto.Query.subquery(q1), + full_join: b in subquery(q2), + on: a.dim0 == b.dim0, + select: %{ + dim0: fragment("coalesce(?, ?)", a.dim0, b.dim0) + } + ) + |> select_joined_metrics(metrics) + end end diff --git a/lib/plausible/stats/metrics.ex b/lib/plausible/stats/metrics.ex index 733c7d6e1c65..8ed71dd2e346 100644 --- a/lib/plausible/stats/metrics.ex +++ b/lib/plausible/stats/metrics.ex @@ -16,6 +16,7 @@ defmodule Plausible.Stats.Metrics do :visit_duration, :events, :conversion_rate, + :group_conversion_rate, :time_on_page ] ++ on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: []) diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index ae25c6536be9..86fc13220c77 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -15,9 +15,10 @@ defmodule Plausible.Stats.Query do experimental_reduced_joins?: false, latest_import_end_date: nil, metrics: [], - order_by: [], + order_by: nil, timezone: nil, - v2: false + v2: false, + preloaded_goals: [] require OpenTelemetry.Tracer, as: Tracer alias Plausible.Stats.{Filters, Interval, Imported} @@ -231,6 +232,18 @@ defmodule Plausible.Stats.Query do |> refresh_imported_opts() end + def set_metrics(query, metrics) do + query + |> struct!(metrics: metrics) + |> refresh_imported_opts() + end + + def set_order_by(query, order_by) do + query + |> struct!(order_by: order_by) + |> refresh_imported_opts() + end + def put_filter(query, filter) do query |> struct!(filters: query.filters ++ [filter]) diff --git a/lib/plausible/stats/query_optimizer.ex b/lib/plausible/stats/query_optimizer.ex index 936fa2758f3e..c1c38e5fb135 100644 --- a/lib/plausible/stats/query_optimizer.ex +++ b/lib/plausible/stats/query_optimizer.ex @@ -1,28 +1,155 @@ defmodule Plausible.Stats.QueryOptimizer do - @moduledoc false + @moduledoc """ + Methods to manipulate Query for business logic reasons before building an ecto query. + """ - alias Plausible.Stats.Query + alias Plausible.Stats.{Query, TableDecider, Util} + @doc """ + This module manipulates an existing query, updating it according to business logic. + + For example, it: + 1. Figures out what the right granularity to group by time is + 2. Adds a missing order_by clause to a query + 3. Updating "time" dimension in order_by to the right granularity + + """ def optimize(query) do Enum.reduce(pipeline(), query, fn step, acc -> step.(acc) end) end + @doc """ + Splits a query into event and sessions subcomponents as not all metrics can be + queried from a single table. + + event:page dimension is treated in a special way, doing a breakdown of visit:entry_page + for sessions. + """ + def split(query) do + {event_metrics, sessions_metrics, _other_metrics} = + query.metrics + |> Util.maybe_add_visitors_metric() + |> TableDecider.partition_metrics(query) + + { + Query.set_metrics(query, event_metrics), + split_sessions_query(query, sessions_metrics) + } + end + defp pipeline() do [ + &update_group_by_time/1, &add_missing_order_by/1, - &update_group_by_time/1 + &update_time_in_order_by/1, + &extend_hostname_filters_to_visit/1 ] end defp add_missing_order_by(%Query{order_by: nil} = query) do - %Query{query | order_by: [{hd(query.metrics), :desc}]} + order_by = + case time_dimension(query) do + nil -> [{hd(query.metrics), :desc}] + time_dimension -> [{time_dimension, :asc}, {hd(query.metrics), :desc}] + end + + %Query{query | order_by: order_by} end defp add_missing_order_by(query), do: query - defp update_group_by_time(%Query{dimensions: ["time" | rest]} = query) do - %Query{query | dimensions: ["time:month" | rest]} + defp update_group_by_time( + %Query{ + date_range: %Date.Range{first: first, last: last} + } = query + ) do + dimensions = + query.dimensions + |> Enum.map(fn + "time" -> resolve_time_dimension(first, last) + entry -> entry + end) + + %Query{query | dimensions: dimensions} end defp update_group_by_time(query), do: query + + defp resolve_time_dimension(first, last) do + cond do + Timex.diff(last, first, :hours) <= 48 -> "time:hour" + Timex.diff(last, first, :days) <= 40 -> "time:day" + true -> "time:month" + end + end + + defp update_time_in_order_by(query) do + order_by = + query.order_by + |> Enum.map(fn + {"time", direction} -> {time_dimension(query), direction} + entry -> entry + end) + + %Query{query | order_by: order_by} + end + + @dimensions_hostname_map %{ + "visit:source" => "visit:entry_page_hostname", + "visit:entry_page" => "visit:entry_page_hostname", + "visit:utm_medium" => "visit:entry_page_hostname", + "visit:utm_source" => "visit:entry_page_hostname", + "visit:utm_campaign" => "visit:entry_page_hostname", + "visit:utm_content" => "visit:entry_page_hostname", + "visit:utm_term" => "visit:entry_page_hostname", + "visit:referrer" => "visit:entry_page_hostname", + "visit:exit_page" => "visit:exit_page_hostname" + } + + # To avoid showing referrers across hostnames when event:hostname + # filter is present for breakdowns, add entry/exit page hostname + # filters + defp extend_hostname_filters_to_visit(query) do + hostname_filters = + query.filters + |> Enum.filter(fn [_operation, filter_key | _rest] -> filter_key == "event:hostname" end) + + if length(hostname_filters) > 0 do + extra_filters = + query.dimensions + |> Enum.flat_map(&hostname_filters_for_dimension(&1, hostname_filters)) + + %Query{query | filters: query.filters ++ extra_filters} + else + query + end + end + + defp hostname_filters_for_dimension(dimension, hostname_filters) do + if Map.has_key?(@dimensions_hostname_map, dimension) do + filter_key = Map.get(@dimensions_hostname_map, dimension) + + hostname_filters + |> Enum.map(fn [operation, _filter_key | rest] -> [operation, filter_key | rest] end) + else + [] + end + end + + defp time_dimension(query) do + Enum.find(query.dimensions, &String.starts_with?(&1, "time")) + end + + defp split_sessions_query(query, session_metrics) do + dimensions = + query.dimensions + |> Enum.map(fn + "event:page" -> "visit:entry_page" + dimension -> dimension + end) + + query + |> Query.set_metrics(session_metrics) + |> Query.set_dimensions(dimensions) + end end diff --git a/lib/plausible/stats/query_result.ex b/lib/plausible/stats/query_result.ex index d7b72bf80d2d..b9f374444834 100644 --- a/lib/plausible/stats/query_result.ex +++ b/lib/plausible/stats/query_result.ex @@ -1,7 +1,7 @@ defmodule Plausible.Stats.QueryResult do @moduledoc false - alias Plausible.Stats.SQL.QueryBuilder + alias Plausible.Stats.Util alias Plausible.Stats.Filters alias Plausible.Stats.Query @@ -15,7 +15,7 @@ defmodule Plausible.Stats.QueryResult do results |> Enum.map(fn entry -> %{ - dimensions: Enum.map(query.dimensions, &Map.get(entry, QueryBuilder.shortname(&1))), + dimensions: Enum.map(query.dimensions, &dimension_label(&1, entry, query)), metrics: Enum.map(query.metrics, &Map.get(entry, &1)) } end) @@ -44,6 +44,22 @@ defmodule Plausible.Stats.QueryResult do defp meta(_), do: %{} + defp dimension_label("event:goal", entry, query) do + {events, paths} = Filters.Utils.split_goals(query.preloaded_goals) + + goal_index = Map.get(entry, Util.shortname(query, "event:goal")) + + # Closely coupled logic with Plausible.Stats.SQL.Expression.event_goal_join/2 + cond do + goal_index < 0 -> Enum.at(events, -goal_index - 1) |> Filters.Utils.unwrap_goal_value() + goal_index > 0 -> Enum.at(paths, goal_index - 1) |> Filters.Utils.unwrap_goal_value() + end + end + + defp dimension_label(dimension, entry, query) do + Map.get(entry, Util.shortname(query, dimension)) + end + defp serializable_filter([operation, "event:goal", clauses]) do [operation, "event:goal", Enum.map(clauses, &Filters.Utils.unwrap_goal_value/1)] end diff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.ex index cdd1cffe4413..e74348daeda9 100644 --- a/lib/plausible/stats/sql/expression.ex +++ b/lib/plausible/stats/sql/expression.ex @@ -1,5 +1,8 @@ defmodule Plausible.Stats.SQL.Expression do - @moduledoc false + @moduledoc """ + This module is responsible for generating SQL/Ecto expressions + for dimensions used in query select, group_by and order_by. + """ import Ecto.Query @@ -86,4 +89,21 @@ defmodule Plausible.Stats.SQL.Expression do def dimension("visit:country", _query), do: dynamic([t], t.country) def dimension("visit:region", _query), do: dynamic([t], t.region) def dimension("visit:city", _query), do: dynamic([t], t.city) + + defmacro event_goal_join(events, page_regexes) do + quote do + fragment( + """ + arrayPushFront( + CAST(multiMatchAllIndices(?, ?) AS Array(Int64)), + -indexOf(?, ?) + ) + """, + e.pathname, + type(^unquote(page_regexes), {:array, :string}), + type(^unquote(events), {:array, :string}), + e.name + ) + end + end end diff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.ex index a966be4b129a..40db95f28f13 100644 --- a/lib/plausible/stats/sql/query_builder.ex +++ b/lib/plausible/stats/sql/query_builder.ex @@ -5,47 +5,45 @@ defmodule Plausible.Stats.SQL.QueryBuilder do import Ecto.Query import Plausible.Stats.Imported + import Plausible.Stats.Util - alias Plausible.Stats.{Base, Query, TableDecider, Util, Filters, Metrics} + alias Plausible.Stats.{Base, Query, QueryOptimizer, TableDecider, Filters, Metrics} alias Plausible.Stats.SQL.Expression + require Plausible.Stats.SQL.Expression + def build(query, site) do - {event_metrics, sessions_metrics, _other_metrics} = - query.metrics - |> Util.maybe_add_visitors_metric() - |> TableDecider.partition_metrics(query) + {event_query, sessions_query} = QueryOptimizer.split(query) + + event_q = build_events_query(site, event_query) + sessions_q = build_sessions_query(site, sessions_query) join_query_results( - build_events_query(site, query, event_metrics), - event_metrics, - build_sessions_query(site, query, sessions_metrics), - sessions_metrics, - query + {event_q, event_query}, + {sessions_q, sessions_query} ) end - def shortname(metric) when is_atom(metric), do: metric - def shortname(dimension), do: Plausible.Stats.Filters.without_prefix(dimension) + defp build_events_query(_site, %Query{metrics: []}), do: nil - defp build_events_query(_, _, []), do: nil - - defp build_events_query(site, query, event_metrics) do + defp build_events_query(site, events_query) do q = from( e in "events_v2", - where: ^Filters.WhereBuilder.build(:events, site, query), - select: ^Base.select_event_metrics(event_metrics) + where: ^Filters.WhereBuilder.build(:events, site, events_query), + select: ^Base.select_event_metrics(events_query.metrics) ) on_ee do - q = Plausible.Stats.Sampling.add_query_hint(q, query) + q = Plausible.Stats.Sampling.add_query_hint(q, events_query) end q - |> join_sessions_if_needed(site, query) - |> build_group_by(query) - |> merge_imported(site, query, event_metrics) - |> Base.maybe_add_conversion_rate(site, query, event_metrics) + |> join_sessions_if_needed(site, events_query) + |> build_group_by(events_query) + |> merge_imported(site, events_query, events_query.metrics) + |> maybe_add_global_conversion_rate(site, events_query) + |> maybe_add_group_conversion_rate(site, events_query) end defp join_sessions_if_needed(q, site, query) do @@ -68,24 +66,24 @@ defmodule Plausible.Stats.SQL.QueryBuilder do end end - def build_sessions_query(_, _, []), do: nil + defp build_sessions_query(_site, %Query{metrics: []}), do: nil - def build_sessions_query(site, query, session_metrics) do + defp build_sessions_query(site, sessions_query) do q = from( e in "sessions_v2", - where: ^Filters.WhereBuilder.build(:sessions, site, query), - select: ^Base.select_session_metrics(session_metrics, query) + where: ^Filters.WhereBuilder.build(:sessions, site, sessions_query), + select: ^Base.select_session_metrics(sessions_query.metrics, sessions_query) ) on_ee do - q = Plausible.Stats.Sampling.add_query_hint(q, query) + q = Plausible.Stats.Sampling.add_query_hint(q, sessions_query) end q - |> join_events_if_needed(site, query) - |> build_group_by(query) - |> merge_imported(site, query, session_metrics) + |> join_events_if_needed(site, sessions_query) + |> build_group_by(sessions_query) + |> merge_imported(site, sessions_query, sessions_query.metrics) end def join_events_if_needed(q, site, query) do @@ -113,15 +111,30 @@ defmodule Plausible.Stats.SQL.QueryBuilder do end defp build_group_by(q, query) do - Enum.reduce(query.dimensions, q, fn dimension, q -> - q - |> select_merge(^%{shortname(dimension) => Expression.dimension(dimension, query)}) - |> group_by(^Expression.dimension(dimension, query)) - end) + Enum.reduce(query.dimensions, q, &dimension_group_by(&2, query, &1)) + end + + defp dimension_group_by(q, query, "event:goal" = dimension) do + {events, page_regexes} = Filters.Utils.split_goals_query_expressions(query.preloaded_goals) + + from(e in q, + array_join: goal in Expression.event_goal_join(events, page_regexes), + select_merge: %{ + ^shortname(query, dimension) => fragment("?", goal) + }, + group_by: goal, + where: goal != 0 + ) + end + + defp dimension_group_by(q, query, dimension) do + q + |> select_merge(^%{shortname(query, dimension) => Expression.dimension(dimension, query)}) + |> group_by(^Expression.dimension(dimension, query)) end defp build_order_by(q, query, mode) do - Enum.reduce(query.order_by, q, &build_order_by(&2, query, &1, mode)) + Enum.reduce(query.order_by || [], q, &build_order_by(&2, query, &1, mode)) end def build_order_by(q, query, {metric_or_dimension, order_direction}, :inner) do @@ -132,36 +145,36 @@ defmodule Plausible.Stats.SQL.QueryBuilder do order_direction, if( Metrics.metric?(metric_or_dimension), - do: dynamic([], selected_as(^shortname(metric_or_dimension))), + do: dynamic([], selected_as(^shortname(query, metric_or_dimension))), else: Expression.dimension(metric_or_dimension, query) ) } ) end - def build_order_by(q, _query, {metric_or_dimension, order_direction}, :outer) do + def build_order_by(q, query, {metric_or_dimension, order_direction}, :outer) do order_by( q, [t], ^{ order_direction, - dynamic([], selected_as(^shortname(metric_or_dimension))) + dynamic([], selected_as(^shortname(query, metric_or_dimension))) } ) end - defmacrop select_join_fields(q, list, table_name) do + defmacrop select_join_fields(q, query, list, table_name) do quote do Enum.reduce(unquote(list), unquote(q), fn metric_or_dimension, q -> select_merge( q, ^%{ - shortname(metric_or_dimension) => + shortname(unquote(query), metric_or_dimension) => dynamic( [e, s], selected_as( - field(unquote(table_name), ^shortname(metric_or_dimension)), - ^shortname(metric_or_dimension) + field(unquote(table_name), ^shortname(unquote(query), metric_or_dimension)), + ^shortname(unquote(query), metric_or_dimension) ) ) } @@ -170,22 +183,98 @@ defmodule Plausible.Stats.SQL.QueryBuilder do end end - defp join_query_results(nil, _, nil, _, _query), do: nil + # Adds conversion_rate metric to query, calculated as + # X / Y where Y is the same breakdown value without goal or props + # filters. + def maybe_add_global_conversion_rate(q, site, query) do + if :conversion_rate in query.metrics do + total_query = + query + |> Query.remove_filters(["event:goal", "event:props"]) + |> Query.set_dimensions([]) + + q + |> select_merge( + ^%{ + total_visitors: Base.total_visitors_subquery(site, total_query, query.include_imported) + } + ) + |> select_merge([e], %{ + conversion_rate: + selected_as( + fragment( + "if(? > 0, round(? / ? * 100, 1), 0)", + selected_as(:__total_visitors), + selected_as(:visitors), + selected_as(:__total_visitors) + ), + :conversion_rate + ) + }) + else + q + end + end + + # This function injects a group_conversion_rate metric into + # a dimensional query. It is calculated as X / Y, where: + # + # * X is the number of conversions for a set of dimensions + # result (conversion = number of visitors who + # completed the filtered goal with the filtered + # custom properties). + # + # * Y is the number of all visitors for this set of dimensions + # result without the `event:goal` and `event:props:*` + # filters. + def maybe_add_group_conversion_rate(q, site, query) do + if :group_conversion_rate in query.metrics do + group_totals_query = + query + |> Query.remove_filters(["event:goal", "event:props"]) + |> Query.set_metrics([:visitors]) + |> Query.set_order_by([]) + + from(e in subquery(q), + left_join: c in subquery(build(group_totals_query, site)), + on: ^build_group_by_join(query), + select_merge: %{ + total_visitors: c.visitors, + group_conversion_rate: + selected_as( + fragment( + "if(? > 0, round(? / ? * 100, 1), 0)", + c.visitors, + e.visitors, + c.visitors + ), + :group_conversion_rate + ) + } + ) + |> select_join_fields(query, query.dimensions, e) + |> select_join_fields(query, List.delete(query.metrics, :group_conversion_rate), e) + else + q + end + end + + defp join_query_results({nil, _}, {nil, _}), do: nil - defp join_query_results(events_q, _, nil, _, query), - do: events_q |> build_order_by(query, :inner) + defp join_query_results({events_q, events_query}, {nil, _}), + do: events_q |> build_order_by(events_query, :inner) - defp join_query_results(nil, _, sessions_q, _, query), - do: sessions_q |> build_order_by(query, :inner) + defp join_query_results({nil, _}, {sessions_q, sessions_query}), + do: sessions_q |> build_order_by(sessions_query, :inner) - defp join_query_results(events_q, event_metrics, sessions_q, sessions_metrics, query) do + defp join_query_results({events_q, events_query}, {sessions_q, sessions_query}) do join(subquery(events_q), :left, [e], s in subquery(sessions_q), - on: ^build_group_by_join(query) + on: ^build_group_by_join(events_query) ) - |> select_join_fields(query.dimensions, e) - |> select_join_fields(event_metrics, e) - |> select_join_fields(List.delete(sessions_metrics, :sample_percent), s) - |> build_order_by(query, :outer) + |> select_join_fields(events_query, events_query.dimensions, e) + |> select_join_fields(events_query, events_query.metrics, e) + |> select_join_fields(sessions_query, List.delete(sessions_query.metrics, :sample_percent), s) + |> build_order_by(events_query, :outer) end defp build_group_by_join(%Query{dimensions: []}), do: true @@ -193,7 +282,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do defp build_group_by_join(query) do query.dimensions |> Enum.map(fn dim -> - dynamic([e, s], field(e, ^shortname(dim)) == field(s, ^shortname(dim))) + dynamic([e, s], field(e, ^shortname(query, dim)) == field(s, ^shortname(query, dim))) end) |> Enum.reduce(fn condition, acc -> dynamic([], ^acc and ^condition) end) end diff --git a/lib/plausible/stats/table_decider.ex b/lib/plausible/stats/table_decider.ex index 629744228b28..c0176d417f6f 100644 --- a/lib/plausible/stats/table_decider.ex +++ b/lib/plausible/stats/table_decider.ex @@ -14,12 +14,6 @@ defmodule Plausible.Stats.TableDecider do |> Enum.any?(&(filters_partitioner(query, &1) == :session)) end - def event_filters?(query) do - query - |> filter_keys() - |> Enum.any?(&(filters_partitioner(query, &1) == :event)) - end - def partition_metrics(metrics, query) do %{ event: event_only_metrics, @@ -64,6 +58,7 @@ defmodule Plausible.Stats.TableDecider do end defp metric_partitioner(_, :conversion_rate), do: :event + defp metric_partitioner(_, :group_conversion_rate), do: :event defp metric_partitioner(_, :average_revenue), do: :event defp metric_partitioner(_, :total_revenue), do: :event defp metric_partitioner(_, :pageviews), do: :event diff --git a/lib/plausible/stats/util.ex b/lib/plausible/stats/util.ex index 85c36888f898..60bdd7a931ab 100644 --- a/lib/plausible/stats/util.ex +++ b/lib/plausible/stats/util.ex @@ -41,7 +41,8 @@ defmodule Plausible.Stats.Util do for any of the other metrics to be calculated. """ def maybe_add_visitors_metric(metrics) do - needed? = Enum.any?([:conversion_rate, :time_on_page], &(&1 in metrics)) + needed? = + Enum.any?([:conversion_rate, :group_conversion_rate, :time_on_page], &(&1 in metrics)) if needed? and :visitors not in metrics do metrics ++ [:visitors] @@ -49,4 +50,12 @@ defmodule Plausible.Stats.Util do metrics end end + + def shortname(_query, metric) when is_atom(metric), do: metric + def shortname(_query, "time:" <> _), do: :time + + def shortname(query, dimension) do + index = Enum.find_index(query.dimensions, &(&1 == dimension)) + :"dim#{index}" + end end diff --git a/test/plausible/stats/query_optimizer_test.exs b/test/plausible/stats/query_optimizer_test.exs new file mode 100644 index 000000000000..aee73b88c0e9 --- /dev/null +++ b/test/plausible/stats/query_optimizer_test.exs @@ -0,0 +1,147 @@ +defmodule Plausible.Stats.QueryOptimizerTest do + use Plausible.DataCase, async: true + + alias Plausible.Stats.{Query, QueryOptimizer} + + @default_params %{metrics: [:visitors]} + + def perform(params) do + params = Map.merge(@default_params, params) |> Map.to_list() + struct!(Query, params) |> QueryOptimizer.optimize() + end + + describe "add_missing_order_by" do + test "does nothing if order_by passed" do + assert perform(%{order_by: [visitors: :desc]}).order_by == [{:visitors, :desc}] + end + + test "adds first metric to order_by if order_by not specified" do + assert perform(%{metrics: [:pageviews, :visitors]}).order_by == [{:pageviews, :desc}] + + assert perform(%{metrics: [:pageviews, :visitors], dimensions: ["event:page"]}).order_by == + [{:pageviews, :desc}] + end + + test "adds time and first metric to order_by if order_by not specified" do + assert perform(%{ + date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-02-01 00:00:00]), + metrics: [:pageviews, :visitors], + dimensions: ["time", "event:page"] + }).order_by == + [{"time:day", :asc}, {:pageviews, :desc}] + end + end + + describe "update_group_by_time" do + test "does nothing if `time` dimension not passed" do + assert perform(%{ + date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-05 00:00:00]), + dimensions: ["time:month"] + }).dimensions == ["time:month"] + end + + test "updating time dimension" do + assert perform(%{ + date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]), + dimensions: ["time"] + }).dimensions == ["time:hour"] + + assert perform(%{ + date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-02 00:00:00]), + dimensions: ["time"] + }).dimensions == ["time:hour"] + + assert perform(%{ + date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-02 16:00:00]), + dimensions: ["time"] + }).dimensions == ["time:hour"] + + assert perform(%{ + date_range: Date.range(~D[2022-01-01], ~D[2022-01-04]), + dimensions: ["time"] + }).dimensions == ["time:day"] + + assert perform(%{ + date_range: Date.range(~D[2022-01-01], ~D[2022-01-10]), + dimensions: ["time"] + }).dimensions == ["time:day"] + + assert perform(%{ + date_range: Date.range(~D[2022-01-01], ~D[2022-01-16]), + dimensions: ["time"] + }).dimensions == ["time:day"] + + assert perform(%{ + date_range: Date.range(~D[2022-01-01], ~D[2022-02-16]), + dimensions: ["time"] + }).dimensions == ["time:month"] + + assert perform(%{ + date_range: Date.range(~D[2022-01-01], ~D[2022-03-16]), + dimensions: ["time"] + }).dimensions == ["time:month"] + + assert perform(%{ + date_range: Date.range(~D[2022-01-01], ~D[2022-03-16]), + dimensions: ["time"] + }).dimensions == ["time:month"] + + assert perform(%{ + date_range: Date.range(~D[2022-01-01], ~D[2023-11-16]), + dimensions: ["time"] + }).dimensions == ["time:month"] + + assert perform(%{ + date_range: Date.range(~D[2022-01-01], ~D[2024-01-16]), + dimensions: ["time"] + }).dimensions == ["time:month"] + + assert perform(%{ + date_range: Date.range(~D[2022-01-01], ~D[2026-01-01]), + dimensions: ["time"] + }).dimensions == ["time:month"] + end + end + + describe "update_time_in_order_by" do + test "updates explicit time dimension in order_by" do + assert perform(%{ + date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]), + dimensions: ["time:hour"], + order_by: [{"time", :asc}] + }).order_by == [{"time:hour", :asc}] + end + end + + describe "extend_hostname_filters_to_visit" do + test "updates filters it filtering by event:hostname and visit:referrer and visit:exit_page dimensions" do + assert perform(%{ + date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]), + filters: [ + [:is, "event:hostname", ["example.com"]], + [:matches, "event:hostname", ["*.com"]] + ], + dimensions: ["visit:referrer", "visit:exit_page"] + }).filters == [ + [:is, "event:hostname", ["example.com"]], + [:matches, "event:hostname", ["*.com"]], + [:is, "visit:entry_page_hostname", ["example.com"]], + [:matches, "visit:entry_page_hostname", ["*.com"]], + [:is, "visit:exit_page_hostname", ["example.com"]], + [:matches, "visit:exit_page_hostname", ["*.com"]] + ] + end + + test "does not update filters if not needed" do + assert perform(%{ + date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]), + filters: [ + [:is, "event:hostname", ["example.com"]] + ], + dimensions: ["time", "event:hostname"] + }).filters == [ + [:is, "event:hostname", ["example.com"]] + ] + end + end +end diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exs index 481f0362dc95..bf3f3bedb492 100644 --- a/test/plausible/stats/query_parser_test.exs +++ b/test/plausible/stats/query_parser_test.exs @@ -6,17 +6,32 @@ defmodule Plausible.Stats.Filters.QueryParserTest do setup [:create_user, :create_new_site] - @date_range Date.range(Timex.today(), Timex.today()) + @today ~D[2021-05-05] + @date_range Date.range(@today, @today) def check_success(params, site, expected_result) do - assert parse(site, params) == {:ok, expected_result} + assert parse(site, params, @today) == {:ok, expected_result} end def check_error(params, site, expected_error_message) do - {:error, message} = parse(site, params) + {:error, message} = parse(site, params, @today) assert message =~ expected_error_message end + def check_date_range(date_range, site, expected_date_range) do + %{"metrics" => ["visitors", "events"], "date_range" => date_range} + |> check_success(site, %{ + metrics: [:visitors, :events], + date_range: expected_date_range, + filters: [], + dimensions: [], + order_by: nil, + timezone: site.timezone, + imported_data_requested: false, + preloaded_goals: [] + }) + end + test "parsing empty map fails", %{site: site} do %{} |> check_error(site, "No valid metrics passed") @@ -32,7 +47,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do dimensions: [], order_by: nil, timezone: site.timezone, - imported_data_requested: false + imported_data_requested: false, + preloaded_goals: [] }) end @@ -69,7 +85,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do dimensions: [], order_by: nil, timezone: site.timezone, - imported_data_requested: false + imported_data_requested: false, + preloaded_goals: [] }) end @@ -98,7 +115,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do dimensions: [], order_by: nil, timezone: site.timezone, - imported_data_requested: false + imported_data_requested: false, + preloaded_goals: [] }) end @@ -142,7 +160,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do dimensions: [], order_by: nil, timezone: site.timezone, - imported_data_requested: false + imported_data_requested: false, + preloaded_goals: [] }) end @@ -165,7 +184,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do dimensions: [], order_by: nil, timezone: site.timezone, - imported_data_requested: false + imported_data_requested: false, + preloaded_goals: [] }) end end @@ -189,7 +209,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do dimensions: [], order_by: nil, timezone: site.timezone, - imported_data_requested: false + imported_data_requested: false, + preloaded_goals: [] }) end end @@ -240,7 +261,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do dimensions: [], order_by: nil, timezone: site.timezone, - imported_data_requested: true + imported_data_requested: true, + preloaded_goals: [] }) end @@ -275,7 +297,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do dimensions: [], order_by: nil, timezone: site.timezone, - imported_data_requested: false + imported_data_requested: false, + preloaded_goals: [{:page, "/thank-you"}, {:event, "Signup"}] }) end @@ -303,6 +326,42 @@ defmodule Plausible.Stats.Filters.QueryParserTest do end describe "date range validation" do + test "parsing shortcut options", %{site: site} do + check_date_range("day", site, Date.range(~D[2021-05-05], ~D[2021-05-05])) + check_date_range("7d", site, Date.range(~D[2021-04-29], ~D[2021-05-05])) + check_date_range("30d", site, Date.range(~D[2021-04-05], ~D[2021-05-05])) + check_date_range("month", site, Date.range(~D[2021-05-01], ~D[2021-05-31])) + check_date_range("6mo", site, Date.range(~D[2020-12-01], ~D[2021-05-31])) + check_date_range("12mo", site, Date.range(~D[2020-06-01], ~D[2021-05-31])) + check_date_range("year", site, Date.range(~D[2021-01-01], ~D[2021-12-31])) + end + + test "parsing `all` with previous data", %{site: site} do + site = Map.put(site, :stats_start_date, ~D[2020-01-01]) + check_date_range("all", site, Date.range(~D[2020-01-01], ~D[2021-05-05])) + end + + test "parsing `all` with no previous data", %{site: site} do + site = Map.put(site, :stats_start_date, nil) + + check_date_range("all", site, Date.range(~D[2021-05-05], ~D[2021-05-05])) + end + + test "parsing custom date range", %{site: site} do + check_date_range( + ["2021-05-05", "2021-05-05"], + site, + Date.range(~D[2021-05-05], ~D[2021-05-05]) + ) + end + + test "parsing invalid custom date range", %{site: site} do + %{"date_range" => "foo", "metrics" => ["visitors"]} + |> check_error(site, ~r/Invalid date_range '\"foo\"'/) + + %{"date_range" => ["21415-00", "eee"], "metrics" => ["visitors"]} + |> check_error(site, ~r/Invalid date_range /) + end end describe "dimensions validation" do @@ -320,7 +379,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do dimensions: ["event:#{unquote(dimension)}"], order_by: nil, timezone: site.timezone, - imported_data_requested: false + imported_data_requested: false, + preloaded_goals: [] }) end end @@ -339,7 +399,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do dimensions: ["visit:#{unquote(dimension)}"], order_by: nil, timezone: site.timezone, - imported_data_requested: false + imported_data_requested: false, + preloaded_goals: [] }) end end @@ -357,7 +418,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do dimensions: ["event:props:foobar"], order_by: nil, timezone: site.timezone, - imported_data_requested: false + imported_data_requested: false, + preloaded_goals: [] }) end @@ -412,7 +474,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do dimensions: [], order_by: [{:events, :desc}, {:visitors, :asc}], timezone: site.timezone, - imported_data_requested: false + imported_data_requested: false, + preloaded_goals: [] }) end @@ -430,7 +493,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do dimensions: ["event:name"], order_by: [{"event:name", :desc}], timezone: site.timezone, - imported_data_requested: false + imported_data_requested: false, + preloaded_goals: [] }) end @@ -525,7 +589,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do dimensions: [], order_by: nil, timezone: site.timezone, - imported_data_requested: false + imported_data_requested: false, + preloaded_goals: [event: "Signup"] }) end @@ -544,11 +609,58 @@ defmodule Plausible.Stats.Filters.QueryParserTest do dimensions: ["event:goal"], order_by: nil, timezone: site.timezone, - imported_data_requested: false + imported_data_requested: false, + preloaded_goals: [event: "Signup"] }) end end + describe "views_per_visit metric" do + test "succeeds with normal filters", %{site: site} do + insert(:goal, %{site: site, event_name: "Signup"}) + + %{ + "metrics" => ["views_per_visit"], + "date_range" => "all", + "filters" => [["is", "event:goal", ["Signup"]]] + } + |> check_success(site, %{ + metrics: [:views_per_visit], + date_range: @date_range, + filters: [[:is, "event:goal", [event: "Signup"]]], + dimensions: [], + order_by: nil, + timezone: site.timezone, + imported_data_requested: false, + preloaded_goals: [event: "Signup"] + }) + end + + test "fails validation if event:page filter specified", %{site: site} do + %{ + "metrics" => ["views_per_visit"], + "date_range" => "all", + "filters" => [["is", "event:page", ["/"]]] + } + |> check_error( + site, + ~r/Metric `views_per_visit` cannot be queried with a filter on `event:page`/ + ) + end + + test "fails validation with dimensions", %{site: site} do + %{ + "metrics" => ["views_per_visit"], + "date_range" => "all", + "dimensions" => ["event:name"] + } + |> check_error( + site, + ~r/Metric `views_per_visit` cannot be queried with `dimensions`/ + ) + end + end + describe "session metrics" do test "single session metric succeeds", %{site: site} do %{ @@ -563,7 +675,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do dimensions: ["visit:device"], order_by: nil, timezone: site.timezone, - imported_data_requested: false + imported_data_requested: false, + preloaded_goals: [] }) end @@ -575,20 +688,44 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_error( site, - "Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions" + "Session metric(s) `bounce_rate` cannot be queried along with event dimensions" ) end - test "fails if using session metric with event filter", %{site: site} do + test "does not fail if using session metric with event:page dimension", %{site: site} do + %{ + "metrics" => ["bounce_rate"], + "date_range" => "all", + "dimensions" => ["event:page"] + } + |> check_success(site, %{ + metrics: [:bounce_rate], + date_range: @date_range, + filters: [], + dimensions: ["event:page"], + order_by: nil, + timezone: site.timezone, + imported_data_requested: false, + preloaded_goals: [] + }) + end + + test "does not fail if using session metric with event filter", %{site: site} do %{ "metrics" => ["bounce_rate"], "date_range" => "all", "filters" => [["is", "event:props:foo", ["(none)"]]] } - |> check_error( - site, - "Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions" - ) + |> check_success(site, %{ + metrics: [:bounce_rate], + date_range: @date_range, + filters: [[:is, "event:props:foo", ["(none)"]]], + dimensions: [], + order_by: nil, + timezone: site.timezone, + imported_data_requested: false, + preloaded_goals: [] + }) end end end diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs index fa6ab457f0d3..cc4aec1ba6a0 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs @@ -141,7 +141,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do "dimensions" => ["event:name"] }) - assert json_response(conn, 400)["error"] =~ "Invalid date range" + assert json_response(conn, 400)["error"] =~ "Invalid date_range" end test "fails when an invalid metric is provided", %{conn: conn, site: site} do @@ -156,7 +156,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do assert json_response(conn, 400)["error"] =~ "Unknown metric '\"baa\"'" end - test "session metrics cannot be used with event:name property", %{conn: conn, site: site} do + test "session metrics cannot be used with event:name dimension", %{conn: conn, site: site} do conn = post(conn, "/api/v2/query", %{ "site_id" => site.domain, @@ -166,10 +166,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do }) assert json_response(conn, 400)["error"] =~ - "Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions" + "Session metric(s) `bounce_rate` cannot be queried along with event dimensions" end - test "session metrics cannot be used with event:props:* property", %{conn: conn, site: site} do + test "session metrics cannot be used with event:props:* dimension", %{conn: conn, site: site} do conn = post(conn, "/api/v2/query", %{ "site_id" => site.domain, @@ -179,23 +179,1103 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do }) assert json_response(conn, 400)["error"] =~ - "Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions" + "Session metric(s) `bounce_rate` cannot be queried along with event dimensions" end - test "session metrics cannot be used with event:name filter", %{conn: conn, site: site} do + test "validates that metric views_per_visit cannot be used with event:page filter", %{ + conn: conn, + site: site + } do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["views_per_visit"], + "filters" => [["is", "event:page", ["/something"]]] + }) + + assert json_response(conn, 400) == %{ + "error" => + "Metric `views_per_visit` cannot be queried with a filter on `event:page`" + } + end + + test "validates that metric views_per_visit cannot be used together with dimensions", %{ + conn: conn, + site: site + } do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["views_per_visit"], + "dimensions" => ["event:name"] + }) + + assert json_response(conn, 400) == %{ + "error" => "Metric `views_per_visit` cannot be queried with `dimensions`" + } + end + + test "validates a metric can't be asked multiple times", %{ + conn: conn, + site: site + } do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["views_per_visit", "visitors", "visitors"] + }) + + assert json_response(conn, 400) == %{ + "error" => "Metrics cannot be queried multiple times" + } + end + end + + test "aggregates a single metric", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["pageviews"], + "date_range" => "all" + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [3], "dimensions" => []}] + end + + test "aggregate views_per_visit rounds to two decimal places", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]), + build(:pageview, user_id: 456, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, user_id: 456, timestamp: ~N[2021-01-01 00:25:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["views_per_visit"], + "date_range" => "all" + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [1.67], "dimensions" => []}] + end + + test "aggregates all metrics in a single query", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => [ + "pageviews", + "visits", + "views_per_visit", + "visitors", + "bounce_rate", + "visit_duration" + ] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [3, 2, 1.5, 2, 50, 750], "dimensions" => []} + ] + end + + describe "aggregation with filters" do + test "can filter by source", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + referrer_source: "Google", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:source", ["Google"]]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end + + test "can filter by no source/referrer", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, + referrer_source: "Google", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:source", ["Direct / None"]]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end + + test "can filter by referrer", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + referrer: "https://facebook.com", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:referrer", ["https://facebook.com"]]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end + + test "wildcard referrer filter with special regex characters", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, referrer: "https://a.com"), + build(:pageview, referrer: "https://a.com"), + build(:pageview, referrer: "https://ab.com") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors"], + "filters" => [ + ["matches", "visit:referrer", ["**a.com**"]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}] + end + + test "can filter by utm_medium", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + utm_medium: "social", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:utm_medium", ["social"]]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end + + test "can filter by utm_source", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + utm_source: "Twitter", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:utm_source", ["Twitter"]]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end + + test "can filter by utm_campaign", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + utm_campaign: "profile", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:utm_campaign", ["profile"]]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end + + test "can filter by device type", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + screen_size: "Desktop", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:device", ["Desktop"]]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end + + test "can filter by browser", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + browser: "Chrome", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:browser", ["Chrome"]]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end + + test "can filter by browser version", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + browser: "Chrome", + browser_version: "56", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:browser_version", ["56"]]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end + + test "can filter by operating system", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + operating_system: "Mac", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:os", ["Mac"]]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end + + test "can filter by operating system version", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + operating_system_version: "10.5", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:os_version", ["10.5"]]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end + + test "can filter by country", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + country_code: "EE", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:country", ["EE"]]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end + + test "can filter by page", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + pathname: "/blogpost", + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, + pathname: "/blogpost", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [ + ["is", "event:page", ["/blogpost"]] + ] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 2, 100, 750], "dimensions" => []} + ] + end + + test "can filter by hostname", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + hostname: "one.example.com", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + hostname: "example.com", + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [ + ["matches", "event:hostname", ["*.example.com", "example.com"]] + ] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 2, 100, 0], "dimensions" => []} + ] + end + + test "filtering by event:name", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Signup", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:event, + name: "Signup", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "pageviews"], + "filters" => [ + ["is", "event:name", ["Signup"]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [2, 0], "dimensions" => []}] + end + + test "filtering by a custom event goal", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Signup", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:event, + name: "Signup", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:event, + name: "NotConfigured", + timestamp: ~N[2021-01-01 00:25:00] + ) + ]) + + insert(:goal, %{site: site, event_name: "Signup"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "events"], + "filters" => [ + ["is", "event:goal", ["Signup"]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [2, 3], "dimensions" => []}] + end + + test "filtering by a revenue goal", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Purchase", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Purchase", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:event, + name: "Purchase", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ) + ]) + + insert(:goal, site: site, currency: :USD, event_name: "Purchase") + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "events"], + "filters" => [ + ["is", "event:goal", ["Purchase"]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [2, 3], "dimensions" => []}] + end + + test "filtering by a simple pageview goal", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/register", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/register", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, + pathname: "/register", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, + pathname: "/", + timestamp: ~N[2021-01-01 00:25:00] + ) + ]) + + insert(:goal, %{site: site, page_path: "/register"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "pageviews"], + "filters" => [ + ["is", "event:goal", ["Visit /register"]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [2, 3], "dimensions" => []}] + end + + test "filtering by a wildcard pageview goal", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/blog/post-1"), + build(:pageview, pathname: "/blog/post-2", user_id: @user_id), + build(:pageview, pathname: "/blog", user_id: @user_id), + build(:pageview, pathname: "/") + ]) + + insert(:goal, %{site: site, page_path: "/blog**"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "pageviews"], + "filters" => [ + ["matches", "event:goal", ["Visit /blog**"]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [2, 3], "dimensions" => []}] + end + + test "filtering by multiple custom event goals", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, name: "Signup"), + build(:event, name: "Purchase", user_id: @user_id), + build(:event, name: "Purchase", user_id: @user_id), + build(:pageview) + ]) + + insert(:goal, %{site: site, event_name: "Signup"}) + insert(:goal, %{site: site, event_name: "Purchase"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "events"], + "filters" => [ + ["is", "event:goal", ["Signup", "Purchase"]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [2, 3], "dimensions" => []}] + end + + test "filtering by multiple mixed goals", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/account/register"), + build(:pageview, pathname: "/register", user_id: @user_id), + build(:event, name: "Signup", user_id: @user_id), + build(:pageview) + ]) + + insert(:goal, %{site: site, event_name: "Signup"}) + insert(:goal, %{site: site, page_path: "/**register"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "events", "pageviews"], + "filters" => [ + ["matches", "event:goal", ["Signup", "Visit /**register"]] + ] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 3, 2], "dimensions" => []} + ] + end + + test "combining filters", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/blogpost", + country_code: "EE", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, + pathname: "/blogpost", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + conn = post(conn, "/api/v2/query", %{ "site_id" => site.domain, - "metrics" => ["visitors", "bounce_rate"], "date_range" => "all", - "dimensions" => ["visit:device"], + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], "filters" => [ - ["is", "event:name", ["pageview"]] + ["is", "event:page", ["/blogpost"]], + ["is", "visit:country", ["EE"]] ] }) - assert json_response(conn, 400)["error"] =~ - "Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions" + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [1, 1, 0, 1500], "dimensions" => []} + ] + end + + test "wildcard page filter", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/en/page1"), + build(:pageview, pathname: "/en/page2"), + build(:pageview, pathname: "/pl/page1") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors"], + "filters" => [ + ["matches", "event:page", ["/en/**"]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}] + end + + test "negated wildcard page filter", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/en/page1"), + build(:pageview, pathname: "/en/page2"), + build(:pageview, pathname: "/pl/page1") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors"], + "filters" => [ + ["does_not_match", "event:page", ["/en/**"]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}] + end + + test "wildcard and member filter combined", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/en/page1"), + build(:pageview, pathname: "/en/page2"), + build(:pageview, pathname: "/pl/page1"), + build(:pageview, pathname: "/ee/page1") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors"], + "filters" => [ + ["matches", "event:page", ["/en/**", "/pl/**"]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [3], "dimensions" => []}] + end + + test "can escape pipe character in member + wildcard filter", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/blog/post|1"), + build(:pageview, pathname: "/otherpost|1"), + build(:pageview, pathname: "/blog/post|2"), + build(:pageview, pathname: "/something-else") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors"], + "filters" => [ + ["matches", "event:page", ["**post\\|1", "/something-else"]] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [3], "dimensions" => []}] + end + + test "handles filtering by visit country", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, country_code: "EE"), + build(:pageview, country_code: "EE"), + build(:pageview, country_code: "EE") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews"], + "filters" => [["is", "visit:country", ["EE"]]] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [3], "dimensions" => []}] + end + end + + describe "aggregation with imported data" do + setup :create_site_import + + test "does not count imported stats unless specified", %{ + conn: conn, + site: site, + site_import: site_import + } do + populate_stats(site, site_import.id, [ + build(:imported_visitors, date: ~D[2023-01-01]), + build(:pageview, timestamp: ~N[2023-01-01 00:00:00]) + ]) + + query_params = %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews"] + } + + conn1 = post(conn, "/api/v2/query", query_params) + + assert json_response(conn1, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}] + + conn2 = post(conn, "/api/v2/query", Map.put(query_params, "include", %{"imports" => true})) + + assert json_response(conn2, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}] + refute json_response(conn2, 200)["meta"]["warning"] + end + end + + describe "timeseries" do + test "shows hourly data for a certain date", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:10:00]), + build(:pageview, timestamp: ~N[2021-01-01 23:59:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "pageviews", "visits", "visit_duration", "bounce_rate"], + "date_range" => ["2021-01-01", "2021-01-01"], + "dimensions" => ["time:hour"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["2021-01-01T00:00:00Z"], "metrics" => [1, 2, 1, 600, 0]}, + %{"dimensions" => ["2021-01-01T23:00:00Z"], "metrics" => [1, 1, 1, 0, 100]} + ] + end + + test "shows last 7 days of visitors", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-07 23:59:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => ["2021-01-01", "2021-01-07"], + "dimensions" => ["time"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-07"], "metrics" => [1]} + ] + end + + test "shows last 6 months of visitors", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2020-08-13 00:00:00]), + build(:pageview, timestamp: ~N[2020-12-31 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => ["2020-07-01", "2021-01-31"], + "dimensions" => ["time"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["2020-08-01"], "metrics" => [1]}, + %{"dimensions" => ["2020-12-01"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-01"], "metrics" => [2]} + ] + end + + test "shows last 12 months of visitors", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2020-02-01 00:00:00]), + build(:pageview, timestamp: ~N[2020-12-31 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => ["2020-01-01", "2021-01-01"], + "dimensions" => ["time"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["2020-02-01"], "metrics" => [1]}, + %{"dimensions" => ["2020-12-01"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-01"], "metrics" => [2]} + ] + end + + test "shows last 12 months of visitors with interval daily", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2020-02-01 00:00:00]), + build(:pageview, timestamp: ~N[2020-12-31 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => ["2020-01-01", "2021-01-07"], + "dimensions" => ["time:day"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["2020-02-01"], "metrics" => [1]}, + %{"dimensions" => ["2020-12-31"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-01"], "metrics" => [2]} + ] + end + + test "shows a custom range with daily interval", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-02 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => ["2021-01-01", "2021-01-02"], + "dimensions" => ["time:day"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["2021-01-01"], "metrics" => [2]}, + %{"dimensions" => ["2021-01-02"], "metrics" => [1]} + ] + end + + test "shows a custom range with monthly interval", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, user_id: @user_id, timestamp: ~N[2020-12-01 00:00:00]), + build(:pageview, user_id: @user_id, timestamp: ~N[2020-12-01 00:05:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-02 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "date_range" => ["2020-12-01", "2021-01-02"], + "dimensions" => ["time:month"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["2020-12-01"], "metrics" => [2, 1, 0, 300]}, + %{"dimensions" => ["2021-01-01"], "metrics" => [2, 2, 100, 0]} + ] + end + + test "timeseries with explicit order_by", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-02 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-02 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-02 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-03 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-03 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-04 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-04 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["pageviews"], + "date_range" => ["2020-12-01", "2021-01-04"], + "dimensions" => ["time"], + "order_by" => [["pageviews", "desc"], ["time", "asc"]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["2021-01-02"], "metrics" => [3]}, + %{"dimensions" => ["2021-01-03"], "metrics" => [2]}, + %{"dimensions" => ["2021-01-04"], "metrics" => [2]}, + %{"dimensions" => ["2021-01-01"], "metrics" => [1]} + ] end end @@ -246,159 +1326,113 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do "dimensions" => ["visit:country"] }) - %{"results" => results} = json_response(conn, 200) - - assert results == [ - %{"dimensions" => ["EE"], "metrics" => [2]}, - %{"dimensions" => ["US"], "metrics" => [1]} + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["EE"], "metrics" => [2]}, + %{"dimensions" => ["US"], "metrics" => [1]} + ] + end + + test "breaks down all metrics by visit:referrer with imported data", %{conn: conn, site: site} do + site_import = + insert(:site_import, + site: site, + start_date: ~D[2005-01-01], + end_date: Timex.today(), + source: :universal_analytics + ) + + populate_stats(site, site_import.id, [ + build(:pageview, referrer: "site.com", timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, referrer: "site.com/1", timestamp: ~N[2021-01-01 00:00:00]), + build(:imported_sources, + referrer: "site.com", + date: ~D[2021-01-01], + visitors: 2, + visits: 2, + pageviews: 2, + bounces: 1, + visit_duration: 120 + ), + build(:imported_sources, + referrer: "site.com/2", + date: ~D[2021-01-01], + visitors: 2, + visits: 2, + pageviews: 2, + bounces: 2, + visit_duration: 0 + ), + build(:imported_sources, + date: ~D[2021-01-01], + visitors: 10, + visits: 11, + pageviews: 50, + bounces: 0, + visit_duration: 1100 + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "visits", "pageviews", "bounce_rate", "visit_duration"], + "date_range" => "all", + "dimensions" => ["visit:referrer"], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Direct / None"], "metrics" => [10, 11, 50, 0.0, 100.0]}, + %{"dimensions" => ["site.com"], "metrics" => [3, 3, 3, 67.0, 40.0]}, + %{"dimensions" => ["site.com/2"], "metrics" => [2, 2, 2, 100.0, 0.0]}, + %{"dimensions" => ["site.com/1"], "metrics" => [1, 1, 1, 100.0, 0.0]} ] end - # test "breaks down all metrics by visit:referrer with imported data", %{conn: conn, site: site} do - # site_import = - # insert(:site_import, - # site: site, - # start_date: ~D[2005-01-01], - # end_date: Timex.today(), - # source: :universal_analytics - # ) - - # populate_stats(site, site_import.id, [ - # build(:pageview, referrer: "site.com", timestamp: ~N[2021-01-01 00:00:00]), - # build(:pageview, referrer: "site.com/1", timestamp: ~N[2021-01-01 00:00:00]), - # build(:imported_sources, - # referrer: "site.com", - # date: ~D[2021-01-01], - # visitors: 2, - # visits: 2, - # pageviews: 2, - # bounces: 1, - # visit_duration: 120 - # ), - # build(:imported_sources, - # referrer: "site.com/2", - # date: ~D[2021-01-01], - # visitors: 2, - # visits: 2, - # pageviews: 2, - # bounces: 2, - # visit_duration: 0 - # ), - # build(:imported_sources, - # date: ~D[2021-01-01], - # visitors: 10, - # visits: 11, - # pageviews: 50, - # bounces: 0, - # visit_duration: 1100 - # ) - # ]) - - # conn = - # get(conn, "/api/v1/stats/breakdown", %{ - # "site_id" => site.domain, - # "period" => "day", - # "metrics" => "visitors,visits,pageviews,bounce_rate,visit_duration", - # "date" => "2021-01-01", - # "property" => "visit:referrer", - # "with_imported" => "true" - # }) - - # assert json_response(conn, 200) == %{ - # "results" => [ - # %{ - # "referrer" => "Direct / None", - # "visitors" => 10, - # "visits" => 11, - # "pageviews" => 50, - # "bounce_rate" => 0, - # "visit_duration" => 100 - # }, - # %{ - # "referrer" => "site.com", - # "visitors" => 3, - # "visits" => 3, - # "pageviews" => 3, - # "bounce_rate" => 67.0, - # "visit_duration" => 40 - # }, - # %{ - # "referrer" => "site.com/2", - # "visitors" => 2, - # "visits" => 2, - # "pageviews" => 2, - # "bounce_rate" => 100.0, - # "visit_duration" => 0 - # }, - # %{ - # "referrer" => "site.com/1", - # "visitors" => 1, - # "visits" => 1, - # "pageviews" => 1, - # "bounce_rate" => 100.0, - # "visit_duration" => 0 - # } - # ] - # } - - # conn = - # post(conn, "/api/v2/query", %{ - # "site_id" => site.domain, - # "metrics" => ["visitors", "visits", "pageviews", "bounce_rate", "visit_duration"], - # "date_range" => "all", - # "dimensions" => ["visit:referrer"], - # }) - - # %{"results" => results} = json_response(conn, 200) - - # assert results == [ - # %{"dimensions" => ["EE"], "metrics" => [2]}, - # %{"dimensions" => ["US"], "metrics" => [1]}, - # ] - # end - - # for {property, attr} <- [ - # {"visit:utm_campaign", :utm_campaign}, - # {"visit:utm_source", :utm_source}, - # {"visit:utm_term", :utm_term}, - # {"visit:utm_content", :utm_content} - # ] do - # test "breakdown by #{property} when filtered by hostname", %{conn: conn, site: site} do - # populate_stats(site, [ - # # session starts at two.example.com with utm_param=ad - # build( - # :pageview, - # [ - # {unquote(attr), "ad"}, - # {:user_id, @user_id}, - # {:hostname, "two.example.com"}, - # {:timestamp, ~N[2021-01-01 00:00:00]} - # ] - # ), - # # session continues on one.example.com without any utm_params - # build( - # :pageview, - # [ - # {:user_id, @user_id}, - # {:hostname, "one.example.com"}, - # {:timestamp, ~N[2021-01-01 00:15:00]} - # ] - # ) - # ]) - - # conn = - # get(conn, "/api/v1/stats/breakdown", %{ - # "site_id" => site.domain, - # "period" => "day", - # "date" => "2021-01-01", - # "filters" => "event:hostname==one.example.com", - # "property" => unquote(property) - # }) - - # # nobody landed on one.example.com from utm_param=ad - # assert json_response(conn, 200) == %{"results" => []} - # end - # end + for {dimension, attr} <- [ + {"visit:utm_campaign", :utm_campaign}, + {"visit:utm_source", :utm_source}, + {"visit:utm_term", :utm_term}, + {"visit:utm_content", :utm_content} + ] do + test "breakdown by #{dimension} when filtered by hostname", %{conn: conn, site: site} do + populate_stats(site, [ + # session starts at two.example.com with utm_param=ad + build( + :pageview, + [ + {unquote(attr), "ad"}, + {:user_id, @user_id}, + {:hostname, "two.example.com"}, + {:timestamp, ~N[2021-01-01 00:00:00]} + ] + ), + # session continues on one.example.com without any utm_params + build( + :pageview, + [ + {:user_id, @user_id}, + {:hostname, "one.example.com"}, + {:timestamp, ~N[2021-01-01 00:15:00]} + ] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["is", "event:hostname", ["one.example.com"]]], + "dimensions" => [unquote(dimension)] + }) + + # nobody landed on one.example.com from utm_param=ad + assert json_response(conn, 200)["results"] == [] + end + end for {dimension, column, value1, value2, blank_value} <- [ {"visit:source", :referrer_source, "Google", "Twitter", "Direct / None"}, @@ -610,125 +1644,101 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do ] end - # test "pageviews breakdown by event:page - imported data having pageviews=0 and visitors=n should be bypassed", - # %{conn: conn, site: site} do - # site_import = - # insert(:site_import, - # site: site, - # start_date: ~D[2005-01-01], - # end_date: Timex.today(), - # source: :universal_analytics - # ) - - # populate_stats(site, site_import.id, [ - # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]), - # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]), - # build(:pageview, - # pathname: "/plausible.io", - # timestamp: ~N[2021-01-01 00:00:00] - # ), - # build(:imported_pages, - # page: "/skip-me", - # date: ~D[2021-01-01], - # visitors: 1, - # pageviews: 0 - # ), - # build(:imported_pages, - # page: "/include-me", - # date: ~D[2021-01-01], - # visitors: 1, - # pageviews: 1 - # ) - # ]) - - # conn = - # get(conn, "/api/v1/stats/breakdown", %{ - # "site_id" => site.domain, - # "period" => "day", - # "date" => "2021-01-01", - # "property" => "event:page", - # "with_imported" => "true", - # "metrics" => "pageviews" - # }) - - # assert json_response(conn, 200) == %{ - # "results" => [ - # %{"page" => "/", "pageviews" => 2}, - # %{"page" => "/plausible.io", "pageviews" => 1}, - # %{"page" => "/include-me", "pageviews" => 1} - # ] - # } - # end - - # test "breakdown by event:page", %{conn: conn, site: site} do - # populate_stats(site, [ - # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]), - # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]), - # build(:pageview, - # pathname: "/plausible.io", - # timestamp: ~N[2021-01-01 00:00:00] - # ) - # ]) - - # conn = - # get(conn, "/api/v1/stats/breakdown", %{ - # "site_id" => site.domain, - # "period" => "day", - # "date" => "2021-01-01", - # "property" => "event:page" - # }) - - # assert json_response(conn, 200) == %{ - # "results" => [ - # %{"page" => "/", "visitors" => 2}, - # %{"page" => "/plausible.io", "visitors" => 1} - # ] - # } - # end - - # test "breakdown by event:page when there are no events in the second page", %{ - # conn: conn, - # site: site - # } do - # populate_stats(site, [ - # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]), - # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]), - # build(:pageview, - # pathname: "/plausible.io", - # timestamp: ~N[2021-01-01 00:00:00] - # ) - # ]) - - # conn = - # get(conn, "/api/v1/stats/breakdown", %{ - # "site_id" => site.domain, - # "period" => "day", - # "date" => "2021-01-01", - # "property" => "event:page", - # "metrics" => "visitors,bounce_rate", - # "page" => 2, - # "limit" => 2 - # }) - - # assert json_response(conn, 200) == %{"results" => []} - # end - - # test "attempting to breakdown by event:hostname returns an error", %{conn: conn, site: site} do - # conn = - # get(conn, "/api/v1/stats/breakdown", %{ - # "site_id" => site.domain, - # "period" => "day", - # "date" => "2021-01-01", - # "property" => "event:hostname", - # "with_imported" => "true" - # }) - - # assert %{ - # "error" => error - # } = json_response(conn, 400) - - # assert error =~ "Property 'event:hostname' is currently not supported for breakdowns." - # end + test "pageviews breakdown by event:page - imported data having pageviews=0 and visitors=n should be bypassed", + %{conn: conn, site: site} do + site_import = + insert(:site_import, + site: site, + start_date: ~D[2005-01-01], + end_date: Timex.today(), + source: :universal_analytics + ) + + populate_stats(site, site_import.id, [ + build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]), + build(:pageview, + pathname: "/plausible.io", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:imported_pages, + page: "/skip-me", + date: ~D[2021-01-01], + visitors: 1, + pageviews: 0 + ), + build(:imported_pages, + page: "/include-me", + date: ~D[2021-01-01], + visitors: 1, + pageviews: 1 + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["pageviews"], + "date_range" => "all", + "dimensions" => ["event:page"], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["/"], "metrics" => [2]}, + %{"dimensions" => ["/plausible.io"], "metrics" => [1]}, + %{"dimensions" => ["/include-me"], "metrics" => [1]} + ] + end + + test "breakdown by event:page", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]), + build(:pageview, + pathname: "/plausible.io", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["/"], "metrics" => [2]}, + %{"dimensions" => ["/plausible.io"], "metrics" => [1]} + ] + end + + test "attempting to breakdown by event:hostname returns an error", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, hostname: "a.example.com"), + build(:pageview, hostname: "a.example.com"), + build(:pageview, hostname: "a.example.com"), + build(:pageview, hostname: "b.example.com") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews"], + "dimensions" => ["event:hostname"], + "with_imported" => "true" + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["a.example.com"], "metrics" => [3]}, + %{"dimensions" => ["b.example.com"], "metrics" => [1]} + ] + end describe "breakdown by visit:exit_page" do setup %{site: site} do @@ -1234,183 +2244,87 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do %{"dimensions" => ["(none)"], "metrics" => [1]} ] end + end + + describe "breakdown by event:goal" do + test "returns custom event goals and pageview goals", %{conn: conn, site: site} do + insert(:goal, %{site: site, event_name: "Purchase"}) + insert(:goal, %{site: site, page_path: "/test"}) + + populate_stats(site, [ + build(:pageview, + timestamp: ~N[2021-01-01 00:00:01], + pathname: "/test" + ), + build(:event, + name: "Purchase", + timestamp: ~N[2021-01-01 00:00:03] + ), + build(:event, + name: "Purchase", + timestamp: ~N[2021-01-01 00:00:03] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors"], + "dimensions" => ["event:goal"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Purchase"], "metrics" => [2]}, + %{"dimensions" => ["Visit /test"], "metrics" => [1]} + ] + end + + test "returns pageview goals containing wildcards", %{conn: conn, site: site} do + insert(:goal, %{site: site, page_path: "/**/post"}) + insert(:goal, %{site: site, page_path: "/blog**"}) + + populate_stats(site, [ + build(:pageview, pathname: "/blog", user_id: @user_id), + build(:pageview, pathname: "/blog/post-1", user_id: @user_id), + build(:pageview, pathname: "/blog/post-2", user_id: @user_id), + build(:pageview, pathname: "/blog/something/post"), + build(:pageview, pathname: "/different/page/post") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "pageviews"], + "dimensions" => ["event:goal"], + "order_by" => [["pageviews", "desc"]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Visit /blog**"], "metrics" => [2, 4]}, + %{"dimensions" => ["Visit /**/post"], "metrics" => [2, 2]} + ] + end + + test "does not return goals that are not configured for the site", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/register"), + build(:event, name: "Signup") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "pageviews"], + "dimensions" => ["event:goal"] + }) - # test "breakdown by custom event property, limited", %{conn: conn, site: site} do - # populate_stats(site, [ - # build(:event, - # name: "Purchase", - # "meta.key": ["cost"], - # "meta.value": ["16"], - # timestamp: ~N[2021-01-01 00:00:00] - # ), - # build(:event, - # name: "Purchase", - # "meta.key": ["cost"], - # "meta.value": ["18"], - # timestamp: ~N[2021-01-01 00:25:00] - # ), - # build(:event, - # name: "Purchase", - # "meta.key": ["cost"], - # "meta.value": ["14"], - # timestamp: ~N[2021-01-01 00:25:00] - # ), - # build(:event, - # name: "Purchase", - # "meta.key": ["cost"], - # "meta.value": ["14"], - # timestamp: ~N[2021-01-01 00:26:00] - # ) - # ]) - - # conn = - # get(conn, "/api/v1/stats/breakdown", %{ - # "site_id" => site.domain, - # "period" => "day", - # "date" => "2021-01-01", - # "property" => "event:props:cost", - # "filters" => "event:name==Purchase", - # "limit" => 2 - # }) - - # assert json_response(conn, 200) == %{ - # "results" => [ - # %{"cost" => "14", "visitors" => 2}, - # %{"cost" => "16", "visitors" => 1} - # ] - # } - # end - - # test "breakdown by custom event property, paginated", %{conn: conn, site: site} do - # populate_stats(site, [ - # build(:event, - # name: "Purchase", - # "meta.key": ["cost"], - # "meta.value": ["16"], - # timestamp: ~N[2021-01-01 00:00:00] - # ), - # build(:event, - # name: "Purchase", - # "meta.key": ["cost"], - # "meta.value": ["16"], - # timestamp: ~N[2021-01-01 00:00:00] - # ), - # build(:event, - # name: "Purchase", - # "meta.key": ["cost"], - # "meta.value": ["18"], - # timestamp: ~N[2021-01-01 00:25:00] - # ), - # build(:event, - # name: "Purchase", - # "meta.key": ["cost"], - # "meta.value": ["14"], - # timestamp: ~N[2021-01-01 00:25:00] - # ), - # build(:event, - # name: "Purchase", - # "meta.key": ["cost"], - # "meta.value": ["14"], - # timestamp: ~N[2021-01-01 00:26:00] - # ) - # ]) - - # conn = - # get(conn, "/api/v1/stats/breakdown", %{ - # "site_id" => site.domain, - # "period" => "day", - # "date" => "2021-01-01", - # "property" => "event:props:cost", - # "filters" => "event:name==Purchase", - # "limit" => 2, - # "page" => 2 - # }) - - # assert json_response(conn, 200) == %{ - # "results" => [ - # %{"cost" => "18", "visitors" => 1} - # ] - # } - # end + assert json_response(conn, 200)["results"] == [] + end end - # describe "breakdown by event:goal" do - # test "returns custom event goals and pageview goals", %{conn: conn, site: site} do - # insert(:goal, %{site: site, event_name: "Purchase"}) - # insert(:goal, %{site: site, page_path: "/test"}) - - # populate_stats(site, [ - # build(:pageview, - # timestamp: ~N[2021-01-01 00:00:01], - # pathname: "/test" - # ), - # build(:event, - # name: "Purchase", - # timestamp: ~N[2021-01-01 00:00:03] - # ), - # build(:event, - # name: "Purchase", - # timestamp: ~N[2021-01-01 00:00:03] - # ) - # ]) - - # conn = - # get(conn, "/api/v1/stats/breakdown", %{ - # "site_id" => site.domain, - # "period" => "day", - # "date" => "2021-01-01", - # "property" => "event:goal" - # }) - - # assert [ - # %{"goal" => "Purchase", "visitors" => 2}, - # %{"goal" => "Visit /test", "visitors" => 1} - # ] = json_response(conn, 200)["results"] - # end - - # test "returns pageview goals containing wildcards", %{conn: conn, site: site} do - # insert(:goal, %{site: site, page_path: "/**/post"}) - # insert(:goal, %{site: site, page_path: "/blog**"}) - - # populate_stats(site, [ - # build(:pageview, pathname: "/blog", user_id: @user_id), - # build(:pageview, pathname: "/blog/post-1", user_id: @user_id), - # build(:pageview, pathname: "/blog/post-2", user_id: @user_id), - # build(:pageview, pathname: "/blog/something/post"), - # build(:pageview, pathname: "/different/page/post") - # ]) - - # conn = - # get(conn, "/api/v1/stats/breakdown", %{ - # "site_id" => site.domain, - # "metrics" => "visitors,pageviews", - # "property" => "event:goal" - # }) - - # assert [ - # %{"goal" => "Visit /**/post", "visitors" => 2, "pageviews" => 2}, - # %{"goal" => "Visit /blog**", "visitors" => 2, "pageviews" => 4} - # ] = json_response(conn, 200)["results"] - # end - - # test "does not return goals that are not configured for the site", %{conn: conn, site: site} do - # populate_stats(site, [ - # build(:pageview, pathname: "/register"), - # build(:event, name: "Signup") - # ]) - - # conn = - # get(conn, "/api/v1/stats/breakdown", %{ - # "site_id" => site.domain, - # "metrics" => "visitors,pageviews", - # "property" => "event:goal" - # }) - - # assert [] = json_response(conn, 200)["results"] - # end - # end - - # describe "filtering" do test "event:goal filter returns 400 when goal not configured", %{conn: conn, site: site} do conn = post(conn, "/api/v2/query", %{ @@ -1456,14 +2370,144 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - pathname: "/plausible.io", - browser: "Chrome", - timestamp: ~N[2021-01-01 00:25:00] + pathname: "/plausible.io", + browser: "Chrome", + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, + browser: "Safari", + pathname: "/plausible.io", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:browser"], + "filters" => [ + ["is", "event:page", ["/plausible.io"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Chrome"], "metrics" => [2]}, + %{"dimensions" => ["Safari"], "metrics" => [1]} + ] + end + + test "event:page filter shows sources of sessions that have visited that page", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + pathname: "/", + referrer_source: "Twitter", + utm_medium: "Twitter", + utm_source: "Twitter", + utm_campaign: "Twitter", + user_id: @user_id + ), + build(:pageview, + pathname: "/plausible.io", + user_id: @user_id + ), + build(:pageview, + pathname: "/plausible.io", + referrer_source: "Google", + utm_medium: "Google", + utm_source: "Google", + utm_campaign: "Google" + ), + build(:pageview, + pathname: "/plausible.io", + referrer_source: "Google", + utm_medium: "Google", + utm_source: "Google", + utm_campaign: "Google" + ) + ]) + + for dimension <- [ + "visit:source", + "visit:utm_medium", + "visit:utm_source", + "visit:utm_campaign" + ] do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => [dimension], + "filters" => [ + ["is", "event:page", ["/plausible.io"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Google"], "metrics" => [2]}, + %{"dimensions" => ["Twitter"], "metrics" => [1]} + ] + end + end + + test "top sources for a custom goal and filtered by hostname", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + hostname: "blog.example.com", + referrer_source: "Facebook", + user_id: @user_id + ), + build(:pageview, + hostname: "app.example.com", + pathname: "/register", + user_id: @user_id + ), + build(:event, + name: "Signup", + hostname: "app.example.com", + pathname: "/register", + user_id: @user_id + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:source"], + "filters" => [ + ["is", "event:hostname", ["app.example.com"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [] + end + + test "top sources for a custom goal and filtered by hostname (2)", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + hostname: "app.example.com", + referrer_source: "Facebook", + pathname: "/register", + user_id: @user_id ), - build(:pageview, - browser: "Safari", - pathname: "/plausible.io", - timestamp: ~N[2021-01-01 00:00:00] + build(:event, + name: "Signup", + hostname: "app.example.com", + pathname: "/register", + user_id: @user_id ) ]) @@ -1472,201 +2516,67 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do "site_id" => site.domain, "metrics" => ["visitors"], "date_range" => "all", - "dimensions" => ["visit:browser"], + "dimensions" => ["visit:source"], "filters" => [ - ["is", "event:page", ["/plausible.io"]] + ["is", "event:hostname", ["app.example.com"]] ] }) - %{"results" => results} = json_response(conn, 200) - - assert results == [ - %{"dimensions" => ["Chrome"], "metrics" => [2]}, - %{"dimensions" => ["Safari"], "metrics" => [1]} + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Facebook"], "metrics" => [1]} ] end - test "event:page filter shows sources of sessions that have visited that page", %{ + test "event:page filter is interpreted as entry_page filter only for bounce_rate", %{ conn: conn, site: site } do populate_stats(site, [ build(:pageview, - pathname: "/", - referrer_source: "Twitter", - utm_medium: "Twitter", - utm_source: "Twitter", - utm_campaign: "Twitter", - user_id: @user_id + browser: "Chrome", + user_id: @user_id, + pathname: "/ignore", + timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, + browser: "Chrome", + user_id: @user_id, pathname: "/plausible.io", - user_id: @user_id + timestamp: ~N[2021-01-01 00:01:00] ), build(:pageview, - pathname: "/plausible.io", - referrer_source: "Google", - utm_medium: "Google", - utm_source: "Google", - utm_campaign: "Google" + browser: "Chrome", + user_id: 456, + pathname: "/important-page", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Chrome", + user_id: 456, + pathname: "/", + timestamp: ~N[2021-01-01 00:01:00] ), build(:pageview, + browser: "Chrome", pathname: "/plausible.io", - referrer_source: "Google", - utm_medium: "Google", - utm_source: "Google", - utm_campaign: "Google" + timestamp: ~N[2021-01-01 00:01:00] ) ]) - for dimension <- [ - "visit:source", - "visit:utm_medium", - "visit:utm_source", - "visit:utm_campaign" - ] do - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "dimensions" => [dimension], - "filters" => [ - ["is", "event:page", ["/plausible.io"]] - ] - }) - - %{"results" => results} = json_response(conn, 200) + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "bounce_rate"], + "filters" => [["is", "event:page", ["/plausible.io", "/important-page"]]], + "dimensions" => ["visit:browser"] + }) - assert results == [ - %{"dimensions" => ["Google"], "metrics" => [2]}, - %{"dimensions" => ["Twitter"], "metrics" => [1]} - ] - end + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Chrome"], "metrics" => [3, 50]} + ] end - # test "top sources for a custom goal and filtered by hostname", %{conn: conn, site: site} do - # populate_stats(site, [ - # build(:pageview, - # hostname: "blog.example.com", - # referrer_source: "Facebook", - # user_id: @user_id - # ), - # build(:pageview, - # hostname: "app.example.com", - # pathname: "/register", - # user_id: @user_id - # ), - # build(:event, - # name: "Signup", - # hostname: "app.example.com", - # pathname: "/register", - # user_id: @user_id - # ) - # ]) - - # conn = - # post(conn, "/api/v2/query", %{ - # "site_id" => site.domain, - # "metrics" => ["visitors"], - # "date_range" => "all", - # "dimensions" => ["visit:source"], - # "filters" => [ - # ["is", "event:hostname", ["app.example.com"]] - # ] - # }) - - # %{"results" => results} = json_response(conn, 200) - - # assert results == [] - # end - - # test "top sources for a custom goal and filtered by hostname (2)", %{conn: conn, site: site} do - # populate_stats(site, [ - # build(:pageview, - # hostname: "app.example.com", - # referrer_source: "Facebook", - # pathname: "/register", - # user_id: @user_id - # ), - # build(:event, - # name: "Signup", - # hostname: "app.example.com", - # pathname: "/register", - # user_id: @user_id - # ) - # ]) - - # conn = - # get(conn, "/api/v1/stats/breakdown", %{ - # "site_id" => site.domain, - # "period" => "day", - # "property" => "visit:source", - # "filters" => "event:hostname==app.example.com" - # }) - - # assert json_response(conn, 200) == %{ - # "results" => [%{"source" => "Facebook", "visitors" => 1}] - # } - # end - - # test "event:page filter is interpreted as entry_page filter only for bounce_rate", %{ - # conn: conn, - # site: site - # } do - # populate_stats(site, [ - # build(:pageview, - # browser: "Chrome", - # user_id: @user_id, - # pathname: "/ignore", - # timestamp: ~N[2021-01-01 00:00:00] - # ), - # build(:pageview, - # browser: "Chrome", - # user_id: @user_id, - # pathname: "/plausible.io", - # timestamp: ~N[2021-01-01 00:01:00] - # ), - # build(:pageview, - # browser: "Chrome", - # user_id: 456, - # pathname: "/important-page", - # timestamp: ~N[2021-01-01 00:00:00] - # ), - # build(:pageview, - # browser: "Chrome", - # user_id: 456, - # pathname: "/", - # timestamp: ~N[2021-01-01 00:01:00] - # ), - # build(:pageview, - # browser: "Chrome", - # pathname: "/plausible.io", - # timestamp: ~N[2021-01-01 00:01:00] - # ) - # ]) - - # conn = - # get(conn, "/api/v1/stats/breakdown", %{ - # "site_id" => site.domain, - # "period" => "day", - # "date" => "2021-01-01", - # "metrics" => "visitors,bounce_rate", - # "property" => "visit:browser", - # "filters" => "event:page == /plausible.io|/important-page" - # }) - - # assert json_response(conn, 200) == %{ - # "results" => [ - # %{ - # "browser" => "Chrome", - # "bounce_rate" => 50, - # "visitors" => 3 - # } - # ] - # } - # end - test "event:goal pageview filter for breakdown by visit source", %{conn: conn, site: site} do insert(:goal, %{site: site, page_path: "/plausible.io"}) @@ -1937,44 +2847,42 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do ] end - # test "IN filter for visit:entry_page", %{conn: conn, site: site} do - # populate_stats(site, [ - # build(:pageview, - # pathname: "/ignore", - # timestamp: ~N[2021-01-01 00:00:00] - # ), - # build(:pageview, - # pathname: "/plausible.io", - # timestamp: ~N[2021-01-01 00:00:00] - # ), - # build(:pageview, - # pathname: "/plausible.io", - # timestamp: ~N[2021-01-01 00:00:00] - # ), - # build(:pageview, - # pathname: "/important-page", - # timestamp: ~N[2021-01-01 00:00:00] - # ) - # ]) - - # conn = - # post(conn, "/api/v2/query", %{ - # "site_id" => site.domain, - # "metrics" => ["bounce_rate"], - # "date_range" => "all", - # "dimensions" => ["event:page"], - # "filters" => [ - # ["is", "event:page", ["/plausible.io", "/important-page"]] - # ] - # }) - - # %{"results" => results} = json_response(conn, 200) - - # assert results == [ - # %{"dimensions" => ["plausible.io"], "metrics" => [100]}, - # %{"dimensions" => ["/important-page"], "metrics" => [100]}, - # ] - # end + test "IN filter for visit:entry_page", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/ignore", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/plausible.io", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/plausible.io", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/important-page", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["bounce_rate"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + ["is", "event:page", ["/plausible.io", "/important-page"]] + ] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["/plausible.io"], "metrics" => [100]}, + %{"dimensions" => ["/important-page"], "metrics" => [100]} + ] + end test "IN filter for event:name", %{conn: conn, site: site} do populate_stats(site, [ @@ -2179,516 +3087,384 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do ] end - # describe "metrics" do - # test "returns conversion_rate in an event:goal breakdown", %{conn: conn, site: site} do - # populate_stats(site, [ - # build(:event, name: "Signup", user_id: 1), - # build(:event, name: "Signup", user_id: 1), - # build(:pageview, pathname: "/blog"), - # build(:pageview, pathname: "/blog/post"), - # build(:pageview) - # ]) - - # insert(:goal, %{site: site, event_name: "Signup"}) - # insert(:goal, %{site: site, page_path: "/blog**"}) - - # conn = - # get(conn, "/api/v1/stats/breakdown", %{ - # "site_id" => site.domain, - # "period" => "day", - # "property" => "event:goal", - # "metrics" => "visitors,events,conversion_rate" - # }) - - # assert json_response(conn, 200) == %{ - # "results" => [ - # %{ - # "goal" => "Visit /blog**", - # "visitors" => 2, - # "events" => 2, - # "conversion_rate" => 50 - # }, - # %{ - # "goal" => "Signup", - # "visitors" => 1, - # "events" => 2, - # "conversion_rate" => 25 - # } - # ] - # } - # end - - # test "returns conversion_rate alone in an event:goal breakdown", %{conn: conn, site: site} do - # populate_stats(site, [ - # build(:event, name: "Signup", user_id: 1), - # build(:pageview) - # ]) - - # insert(:goal, %{site: site, event_name: "Signup"}) - - # conn = - # get(conn, "/api/v1/stats/breakdown", %{ - # "site_id" => site.domain, - # "period" => "day", - # "property" => "event:goal", - # "metrics" => "conversion_rate" - # }) - - # assert json_response(conn, 200) == %{ - # "results" => [ - # %{ - # "goal" => "Signup", - # "conversion_rate" => 50 - # } - # ] - # } - # end - - # test "returns conversion_rate in a goal filtered custom prop breakdown", %{ - # conn: conn, - # site: site - # } do - # populate_stats(site, [ - # build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]), - # build(:pageview, pathname: "/blog/2", "meta.key": ["author"], "meta.value": ["Uku"]), - # build(:pageview, pathname: "/blog/3", "meta.key": ["author"], "meta.value": ["Uku"]), - # build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Marko"]), - # build(:pageview, - # pathname: "/blog/2", - # "meta.key": ["author"], - # "meta.value": ["Marko"], - # user_id: 1 - # ), - # build(:pageview, - # pathname: "/blog/3", - # "meta.key": ["author"], - # "meta.value": ["Marko"], - # user_id: 1 - # ), - # build(:pageview, pathname: "/blog"), - # build(:pageview, "meta.key": ["author"], "meta.value": ["Marko"]), - # build(:pageview) - # ]) - - # insert(:goal, %{site: site, page_path: "/blog**"}) - - # conn = - # get(conn, "/api/v1/stats/breakdown", %{ - # "site_id" => site.domain, - # "period" => "day", - # "property" => "event:props:author", - # "filters" => "event:goal==Visit /blog**", - # "metrics" => "visitors,events,conversion_rate" - # }) - - # assert json_response(conn, 200) == %{ - # "results" => [ - # %{ - # "author" => "Uku", - # "visitors" => 3, - # "events" => 3, - # "conversion_rate" => 37.5 - # }, - # %{ - # "author" => "Marko", - # "visitors" => 2, - # "events" => 3, - # "conversion_rate" => 25 - # }, - # %{ - # "author" => "(none)", - # "visitors" => 1, - # "events" => 1, - # "conversion_rate" => 12.5 - # } - # ] - # } - # end - - test "returns conversion_rate alone in a goal filtered custom prop breakdown", %{ - conn: conn, - site: site - } do - populate_stats(site, [ - build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]), - build(:pageview) - ]) + describe "metrics" do + test "returns conversion_rate in an event:goal breakdown", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, name: "Signup", user_id: 1), + build(:event, name: "Signup", user_id: 1), + build(:pageview, pathname: "/blog"), + build(:pageview, pathname: "/blog/post"), + build(:pageview) + ]) - insert(:goal, %{site: site, page_path: "/blog**"}) + insert(:goal, %{site: site, event_name: "Signup"}) + insert(:goal, %{site: site, page_path: "/blog**"}) - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["conversion_rate"], - "date_range" => "all", - "dimensions" => ["event:props:author"], - "filters" => [["matches", "event:goal", ["Visit /blog**"]]] - }) + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "events", "conversion_rate"], + "dimensions" => ["event:goal"] + }) - %{"results" => results} = json_response(conn, 200) + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Visit /blog**"], "metrics" => [2, 2, 50.0]}, + %{"dimensions" => ["Signup"], "metrics" => [1, 2, 25.0]} + ] + end - assert results == [ - %{"dimensions" => ["Uku"], "metrics" => [50]} - ] - end + test "returns conversion_rate alone in an event:goal breakdown", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, name: "Signup", user_id: 1), + build(:pageview) + ]) - # test "returns conversion_rate in a goal filtered event:page breakdown", %{ - # conn: conn, - # site: site - # } do - # populate_stats(site, [ - # build(:event, pathname: "/en/register", name: "pageview"), - # build(:event, pathname: "/en/register", name: "Signup"), - # build(:event, pathname: "/en/register", name: "Signup"), - # build(:event, pathname: "/it/register", name: "Signup", user_id: 1), - # build(:event, pathname: "/it/register", name: "Signup", user_id: 1), - # build(:event, pathname: "/it/register", name: "pageview") - # ]) - - # insert(:goal, %{site: site, event_name: "Signup"}) - - # conn = - # get(conn, "/api/v1/stats/breakdown", %{ - # "site_id" => site.domain, - # "period" => "day", - # "property" => "event:page", - # "filters" => "event:goal==Signup", - # "metrics" => "visitors,events,conversion_rate" - # }) - - # assert json_response(conn, 200) == %{ - # "results" => [ - # %{ - # "page" => "/en/register", - # "visitors" => 2, - # "events" => 2, - # "conversion_rate" => 66.7 - # }, - # %{ - # "page" => "/it/register", - # "visitors" => 1, - # "events" => 2, - # "conversion_rate" => 50 - # } - # ] - # } - # end - - # test "returns conversion_rate alone in a goal filtered event:page breakdown", %{ - # conn: conn, - # site: site - # } do - # populate_stats(site, [ - # build(:event, pathname: "/en/register", name: "pageview"), - # build(:event, pathname: "/en/register", name: "Signup") - # ]) - - # insert(:goal, %{site: site, event_name: "Signup"}) - - # conn = - # get(conn, "/api/v1/stats/breakdown", %{ - # "site_id" => site.domain, - # "period" => "day", - # "property" => "event:page", - # "filters" => "event:goal==Signup", - # "metrics" => "conversion_rate" - # }) - - # assert json_response(conn, 200) == %{ - # "results" => [ - # %{ - # "page" => "/en/register", - # "conversion_rate" => 50 - # } - # ] - # } - # end - - # test "returns conversion_rate in a multi-goal filtered visit:screen_size breakdown", %{ - # conn: conn, - # site: site - # } do - # populate_stats(site, [ - # build(:event, screen_size: "Mobile", name: "pageview"), - # build(:event, screen_size: "Mobile", name: "AddToCart"), - # build(:event, screen_size: "Mobile", name: "AddToCart"), - # build(:event, screen_size: "Desktop", name: "AddToCart", user_id: 1), - # build(:event, screen_size: "Desktop", name: "Purchase", user_id: 1), - # build(:event, screen_size: "Desktop", name: "pageview") - # ]) - - # # Make sure that revenue goals are treated the same - # # way as regular custom event goals - # insert(:goal, %{site: site, event_name: "Purchase", currency: :EUR}) - # insert(:goal, %{site: site, event_name: "AddToCart"}) - - # conn = - # post(conn, "/api/v2/query", %{ - # "site_id" => site.domain, - # "metrics" => ["visitors", "events", "conversion_rate"], - # "date_range" => "all", - # "dimensions" => ["visit:device"], - # "filters" => [["is", "event:goal", ["AddToCart", "Purchase"]]] - # }) - - # %{"results" => results} = json_response(conn, 200) - - # assert results == [ - # %{"dimensions" => ["Mobile"], "metrics" => [2, 2, 66.7]}, - # %{"dimensions" => ["Desktop"], "metrics" => [1, 2, 50]}, - # ] - # end - - test "returns conversion_rate alone in a goal filtered visit:screen_size breakdown", %{ - conn: conn, - site: site - } do - populate_stats(site, [ - build(:event, screen_size: "Mobile", name: "pageview"), - build(:event, screen_size: "Mobile", name: "AddToCart") - ]) + insert(:goal, %{site: site, event_name: "Signup"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["conversion_rate"], + "dimensions" => ["event:goal"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Signup"], "metrics" => [50.0]} + ] + end + + test "returns conversion_rate in a goal filtered custom prop breakdown", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]), + build(:pageview, pathname: "/blog/2", "meta.key": ["author"], "meta.value": ["Uku"]), + build(:pageview, pathname: "/blog/3", "meta.key": ["author"], "meta.value": ["Uku"]), + build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Marko"]), + build(:pageview, + pathname: "/blog/2", + "meta.key": ["author"], + "meta.value": ["Marko"], + user_id: 1 + ), + build(:pageview, + pathname: "/blog/3", + "meta.key": ["author"], + "meta.value": ["Marko"], + user_id: 1 + ), + build(:pageview, pathname: "/blog"), + build(:pageview, "meta.key": ["author"], "meta.value": ["Marko"]), + build(:pageview) + ]) + + insert(:goal, %{site: site, page_path: "/blog**"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "filters" => [["matches", "event:goal", ["Visit /blog**"]]], + "metrics" => ["visitors", "events", "conversion_rate"], + "dimensions" => ["event:props:author"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Uku"], "metrics" => [3, 3, 37.5]}, + %{"dimensions" => ["Marko"], "metrics" => [2, 3, 25.0]}, + %{"dimensions" => ["(none)"], "metrics" => [1, 1, 12.5]} + ] + end + + test "returns conversion_rate alone in a goal filtered custom prop breakdown", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]), + build(:pageview) + ]) + + insert(:goal, %{site: site, page_path: "/blog**"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["conversion_rate"], + "date_range" => "all", + "dimensions" => ["event:props:author"], + "filters" => [["matches", "event:goal", ["Visit /blog**"]]] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Uku"], "metrics" => [50]} + ] + end + + test "returns conversion_rate in a goal filtered event:page breakdown", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, pathname: "/en/register", name: "pageview"), + build(:event, pathname: "/en/register", name: "Signup"), + build(:event, pathname: "/en/register", name: "Signup"), + build(:event, pathname: "/it/register", name: "Signup", user_id: 1), + build(:event, pathname: "/it/register", name: "Signup", user_id: 1), + build(:event, pathname: "/it/register", name: "pageview") + ]) + + insert(:goal, %{site: site, event_name: "Signup"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [["is", "event:goal", ["Signup"]]], + "metrics" => ["visitors", "events", "group_conversion_rate"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["/en/register"], "metrics" => [2, 2, 66.7]}, + %{"dimensions" => ["/it/register"], "metrics" => [1, 2, 50.0]} + ] + end + + test "returns conversion_rate alone in a goal filtered event:page breakdown", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, pathname: "/en/register", name: "pageview"), + build(:event, pathname: "/en/register", name: "Signup") + ]) + + insert(:goal, %{site: site, event_name: "Signup"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "filters" => [["is", "event:goal", ["Signup"]]], + "metrics" => ["group_conversion_rate"], + "dimensions" => ["event:page"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["/en/register"], "metrics" => [50.0]} + ] + end + + test "returns conversion_rate in a multi-goal filtered visit:screen_size breakdown", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, screen_size: "Mobile", name: "pageview"), + build(:event, screen_size: "Mobile", name: "AddToCart"), + build(:event, screen_size: "Mobile", name: "AddToCart"), + build(:event, screen_size: "Desktop", name: "AddToCart", user_id: 1), + build(:event, screen_size: "Desktop", name: "Purchase", user_id: 1), + build(:event, screen_size: "Desktop", name: "pageview") + ]) + + # Make sure that revenue goals are treated the same + # way as regular custom event goals + insert(:goal, %{site: site, event_name: "Purchase", currency: :EUR}) + insert(:goal, %{site: site, event_name: "AddToCart"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "events", "group_conversion_rate"], + "date_range" => "all", + "dimensions" => ["visit:device"], + "filters" => [["is", "event:goal", ["AddToCart", "Purchase"]]] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Mobile"], "metrics" => [2, 2, 66.7]}, + %{"dimensions" => ["Desktop"], "metrics" => [1, 2, 50]} + ] + end + + test "returns conversion_rate alone in a goal filtered visit:screen_size breakdown", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, screen_size: "Mobile", name: "pageview"), + build(:event, screen_size: "Mobile", name: "AddToCart") + ]) - insert(:goal, %{site: site, event_name: "AddToCart"}) + insert(:goal, %{site: site, event_name: "AddToCart"}) - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["conversion_rate"], - "date_range" => "all", - "dimensions" => ["visit:device"], - "filters" => [["is", "event:goal", ["AddToCart"]]] - }) + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["conversion_rate"], + "date_range" => "all", + "dimensions" => ["visit:device"], + "filters" => [["is", "event:goal", ["AddToCart"]]] + }) - %{"results" => results} = json_response(conn, 200) + %{"results" => results} = json_response(conn, 200) - assert results == [ - %{"dimensions" => ["Mobile"], "metrics" => [50]} - ] - end + assert results == [ + %{"dimensions" => ["Mobile"], "metrics" => [50]} + ] + end - # test "returns conversion_rate for a browser_version breakdown with pagination limit", %{ - # site: site, - # conn: conn - # } do - # populate_stats(site, [ - # build(:pageview, browser: "Firefox", browser_version: "110"), - # build(:pageview, browser: "Firefox", browser_version: "110"), - # build(:pageview, browser: "Chrome", browser_version: "110"), - # build(:pageview, browser: "Chrome", browser_version: "110"), - # build(:pageview, browser: "Avast Secure Browser", browser_version: "110"), - # build(:pageview, browser: "Avast Secure Browser", browser_version: "110"), - # build(:event, name: "Signup", browser: "Edge", browser_version: "110") - # ]) - - # insert(:goal, site: site, event_name: "Signup") - - # conn = - # get(conn, "/api/v1/stats/breakdown", %{ - # "site_id" => site.domain, - # "period" => "day", - # "property" => "visit:browser_version", - # "filters" => "event:goal==Signup", - # "metrics" => "visitors,conversion_rate", - # "page" => 1, - # "limit" => 1 - # }) - - # assert json_response(conn, 200) == %{ - # "results" => [ - # %{ - # "browser" => "Edge", - # "browser_version" => "110", - # "visitors" => 1, - # "conversion_rate" => 100.0 - # } - # ] - # } - # end - - test "all metrics for breakdown by visit prop", %{conn: conn, site: site} do - populate_stats(site, [ - build(:pageview, - user_id: 1, - referrer_source: "Google", - timestamp: ~N[2021-01-01 00:00:00] - ), - build(:event, - name: "signup", - user_id: 1, - referrer_source: "Google", - timestamp: ~N[2021-01-01 00:05:00] - ), - build(:pageview, - user_id: 1, - referrer_source: "Google", - timestamp: ~N[2021-01-01 00:10:00] - ), - build(:pageview, - referrer_source: "Google", - timestamp: ~N[2021-01-01 00:25:00] - ), - build(:pageview, - referrer_source: "Twitter", - timestamp: ~N[2021-01-01 00:00:00] - ) - ]) + test "all metrics for breakdown by visit prop", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + user_id: 1, + referrer_source: "Google", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "signup", + user_id: 1, + referrer_source: "Google", + timestamp: ~N[2021-01-01 00:05:00] + ), + build(:pageview, + user_id: 1, + referrer_source: "Google", + timestamp: ~N[2021-01-01 00:10:00] + ), + build(:pageview, + referrer_source: "Google", + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, + referrer_source: "Twitter", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => [ - "visitors", - "visits", - "pageviews", - "events", - "bounce_rate", - "visit_duration" - ], - "date_range" => "all", - "dimensions" => ["visit:source"], - "filters" => [] - }) + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => [ + "visitors", + "visits", + "pageviews", + "events", + "bounce_rate", + "visit_duration" + ], + "date_range" => "all", + "dimensions" => ["visit:source"], + "filters" => [] + }) - %{"results" => results} = json_response(conn, 200) + %{"results" => results} = json_response(conn, 200) - assert results == [ - %{"dimensions" => ["Google"], "metrics" => [2, 2, 3, 4, 50, 300]}, - %{"dimensions" => ["Twitter"], "metrics" => [1, 1, 1, 1, 100, 0]} - ] - end + assert results == [ + %{"dimensions" => ["Google"], "metrics" => [2, 2, 3, 4, 50, 300]}, + %{"dimensions" => ["Twitter"], "metrics" => [1, 1, 1, 1, 100, 0]} + ] + end - test "metrics=bounce_rate does not add visits to the response", %{conn: conn, site: site} do - populate_stats(site, [ - build(:pageview, - user_id: 1, - pathname: "/entry-page-1", - timestamp: ~N[2021-01-01 00:00:00] - ), - build(:pageview, - user_id: 1, - pathname: "/some-page", - timestamp: ~N[2021-01-01 00:10:00] - ), - build(:pageview, - user_id: 2, - pathname: "/entry-page-2", - referrer_source: "Google", - timestamp: ~N[2021-01-01 00:05:00] - ) - ]) + test "metrics=bounce_rate does not add visits to the response", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + user_id: 1, + pathname: "/entry-page-1", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: 1, + pathname: "/some-page", + timestamp: ~N[2021-01-01 00:10:00] + ), + build(:pageview, + user_id: 2, + pathname: "/entry-page-2", + referrer_source: "Google", + timestamp: ~N[2021-01-01 00:05:00] + ) + ]) - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["bounce_rate"], - "date_range" => "all", - "dimensions" => ["visit:entry_page"], - "order_by" => [["visit:entry_page", "asc"]] - }) + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["bounce_rate"], + "date_range" => "all", + "dimensions" => ["visit:entry_page"], + "order_by" => [["visit:entry_page", "asc"]] + }) - %{"results" => results} = json_response(conn, 200) + %{"results" => results} = json_response(conn, 200) - assert results == [ - %{"dimensions" => ["/entry-page-1"], "metrics" => [0]}, - %{"dimensions" => ["/entry-page-2"], "metrics" => [100]} - ] - end + assert results == [ + %{"dimensions" => ["/entry-page-1"], "metrics" => [0]}, + %{"dimensions" => ["/entry-page-2"], "metrics" => [100]} + ] + end - test "filter by custom event property", %{conn: conn, site: site} do - populate_stats(site, [ - build(:event, - name: "Purchase", - "meta.key": ["package"], - "meta.value": ["business"], - browser: "Chrome", - timestamp: ~N[2021-01-01 00:00:00] - ), - build(:event, - name: "Purchase", - "meta.key": ["package"], - "meta.value": ["business"], - browser: "Safari", - timestamp: ~N[2021-01-01 00:00:00] - ), - build(:event, - name: "Purchase", - "meta.key": ["package"], - "meta.value": ["business"], - browser: "Safari", - timestamp: ~N[2021-01-01 00:25:00] - ), - build(:event, - name: "Purchase", - "meta.key": ["package"], - "meta.value": ["personal"], - browser: "IE", - timestamp: ~N[2021-01-01 00:25:00] - ) - ]) + test "filter by custom event property", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Purchase", + "meta.key": ["package"], + "meta.value": ["business"], + browser: "Chrome", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Purchase", + "meta.key": ["package"], + "meta.value": ["business"], + browser: "Safari", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Purchase", + "meta.key": ["package"], + "meta.value": ["business"], + browser: "Safari", + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:event, + name: "Purchase", + "meta.key": ["package"], + "meta.value": ["personal"], + browser: "IE", + timestamp: ~N[2021-01-01 00:25:00] + ) + ]) - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "dimensions" => ["visit:browser"], - "filters" => [ - ["is", "event:name", ["Purchase"]], - ["is", "event:props:package", ["business"]] - ] - }) + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:browser"], + "filters" => [ + ["is", "event:name", ["Purchase"]], + ["is", "event:props:package", ["business"]] + ] + }) - %{"results" => results} = json_response(conn, 200) + %{"results" => results} = json_response(conn, 200) - assert results == [ - %{"dimensions" => ["Safari"], "metrics" => [2]}, - %{"dimensions" => ["Chrome"], "metrics" => [1]} - ] + assert results == [ + %{"dimensions" => ["Safari"], "metrics" => [2]}, + %{"dimensions" => ["Chrome"], "metrics" => [1]} + ] + end end - # test "all metrics for breakdown by event prop", %{conn: conn, site: site} do - # populate_stats(site, [ - # build(:pageview, - # user_id: 1, - # pathname: "/", - # timestamp: ~N[2021-01-01 00:00:00] - # ), - # build(:pageview, - # user_id: 1, - # pathname: "/plausible.io", - # timestamp: ~N[2021-01-01 00:10:00] - # ), - # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]), - # build(:pageview, - # pathname: "/plausible.io", - # timestamp: ~N[2021-01-01 00:00:00] - # ) - # ]) - - # conn = - # post(conn, "/api/v2/query", %{ - # "site_id" => site.domain, - # "metrics" => [ - # "visitors", - # "visits", - # "pageviews", - # "events", - # "bounce_rate", - # "visit_duration" - # ], - # "date_range" => "all", - # "dimensions" => ["event:page"], - # }) - - # %{"results" => results} = json_response(conn, 200) - - # assert results == [ - # %{"dimensions" => ["/"], "metrics" => [2, 2, 3, 4, 50, 300]}, - # %{"dimensions" => ["/plausible.io"], "metrics" => [1, 1, 1, 1, 100, 0]} - # ] - # end - # end - describe "imported data" do test "returns screen sizes breakdown when filtering by screen size", %{conn: conn, site: site} do site_import = insert(:site_import, site: site) @@ -2723,67 +3499,54 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do assert results == [%{"dimensions" => ["Mobile"], "metrics" => [4, 6]}] end - # test "returns custom event goals and pageview goals", %{conn: conn, site: site} do - # insert(:goal, site: site, event_name: "Purchase") - # insert(:goal, site: site, page_path: "/test") - - # site_import = insert(:site_import, site: site) - - # populate_stats(site, site_import.id, [ - # build(:pageview, - # timestamp: ~N[2021-01-01 00:00:01], - # pathname: "/test" - # ), - # build(:event, - # name: "Purchase", - # timestamp: ~N[2021-01-01 00:00:03] - # ), - # build(:event, - # name: "Purchase", - # timestamp: ~N[2021-01-01 00:00:03] - # ), - # build(:imported_custom_events, - # name: "Purchase", - # visitors: 3, - # events: 5, - # date: ~D[2021-01-01] - # ), - # build(:imported_pages, - # page: "/test", - # visitors: 2, - # pageviews: 2, - # date: ~D[2021-01-01] - # ), - # build(:imported_visitors, visitors: 5, date: ~D[2021-01-01]) - # ]) - - # conn = - # get(conn, "/api/v1/stats/breakdown", %{ - # "site_id" => site.domain, - # "period" => "day", - # "date" => "2021-01-01", - # "property" => "event:goal", - # "metrics" => "visitors,events,pageviews,conversion_rate", - # "with_imported" => "true" - # }) - - # assert [ - # %{ - # "goal" => "Purchase", - # "visitors" => 5, - # "events" => 7, - # "pageviews" => 0, - # "conversion_rate" => 62.5 - # }, - # %{ - # "goal" => "Visit /test", - # "visitors" => 3, - # "events" => 3, - # "pageviews" => 3, - # "conversion_rate" => 37.5 - # } - # ] = json_response(conn, 200)["results"] - # end + test "returns custom event goals and pageview goals", %{conn: conn, site: site} do + insert(:goal, site: site, event_name: "Purchase") + insert(:goal, site: site, page_path: "/test") + + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:pageview, + timestamp: ~N[2021-01-01 00:00:01], + pathname: "/test" + ), + build(:event, + name: "Purchase", + timestamp: ~N[2021-01-01 00:00:03] + ), + build(:event, + name: "Purchase", + timestamp: ~N[2021-01-01 00:00:03] + ), + build(:imported_custom_events, + name: "Purchase", + visitors: 3, + events: 5, + date: ~D[2021-01-01] + ), + build(:imported_pages, + page: "/test", + visitors: 2, + pageviews: 2, + date: ~D[2021-01-01] + ), + build(:imported_visitors, visitors: 5, date: ~D[2021-01-01]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "dimensions" => ["event:goal"], + "metrics" => ["visitors", "events", "pageviews", "conversion_rate"], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Purchase"], "metrics" => [5, 7, 0, 62.5]}, + %{"dimensions" => ["Visit /test"], "metrics" => [3, 3, 3, 37.5]} + ] + end test "pageviews are returned as events for breakdown reports other than custom events", %{ conn: conn, @@ -2826,6 +3589,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do assert %{"dimensions" => ["Desktop"], "metrics" => [1]} = breakdown_and_first.("visit:device") + # :TODO: These should not pass validation - not available on events. + # visit dimension and event-only metric # assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("visit:entry_page") # assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("visit:exit_page") assert %{"dimensions" => ["EE"], "metrics" => [1]} = breakdown_and_first.("visit:country") @@ -2870,8 +3635,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do conn = post(conn, "/api/v2/query", %{ "site_id" => site.domain, - "metrics" => ["visitors", "events"], - # "metrics" => ["visitors", "events", "conversion_rate"], + "metrics" => ["visitors", "events", "conversion_rate"], "date_range" => "all", "dimensions" => ["event:props:url"], "filters" => [ @@ -2881,8 +3645,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do }) assert json_response(conn, 200)["results"] == [ - %{"dimensions" => ["https://two.com"], "metrics" => [5, 10]}, - %{"dimensions" => ["https://one.com"], "metrics" => [3, 6]} + %{"dimensions" => ["https://two.com"], "metrics" => [5, 10, 50]}, + %{"dimensions" => ["https://one.com"], "metrics" => [3, 6, 30]} ] refute json_response(conn, 200)["meta"]["warning"] @@ -2923,8 +3687,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do conn = post(conn, "/api/v2/query", %{ "site_id" => site.domain, - "metrics" => ["visitors", "events"], - # "metrics" => ["visitors", "events", "conversion_rate"], + "metrics" => ["visitors", "events", "conversion_rate"], "date_range" => "all", "dimensions" => ["event:props:path"], "filters" => [ @@ -2934,8 +3697,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do }) assert json_response(conn, 200)["results"] == [ - %{"dimensions" => ["/two"], "metrics" => [5, 10]}, - %{"dimensions" => ["/one"], "metrics" => [3, 6]} + %{"dimensions" => ["/two"], "metrics" => [5, 10, 50]}, + %{"dimensions" => ["/one"], "metrics" => [3, 6, 30]} ] refute json_response(conn, 200)["meta"]["warning"] @@ -3103,4 +3866,39 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do assert meta["warning"] =~ "Imported stats are not included in the results" end end + + test "multiple breakdown timeseries with sources", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + referrer_source: "Google", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, timestamp: ~N[2021-01-02 00:00:00]), + build(:pageview, + referrer_source: "Google", + timestamp: ~N[2021-01-02 00:00:00] + ), + build(:pageview, timestamp: ~N[2021-01-03 00:00:00]), + build(:pageview, + referrer_source: "Twitter", + timestamp: ~N[2021-01-03 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => ["2021-01-01", "2021-01-03"], + "dimensions" => ["time", "visit:source"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["2021-01-01T00:00:00Z", "Google"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-02T00:00:00Z", "Google"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-02T00:00:00Z", "Direct / None"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-03T00:00:00Z", "Direct / None"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-03T00:00:00Z", "Twitter"], "metrics" => [1]} + ] + end end