diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 1442c9013f0a..9cdb024571c5 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -7,7 +7,7 @@ defmodule Plausible.Google.API do alias Plausible.Google.HTTP alias Plausible.Google.SearchConsole - alias Plausible.Stats.DateTimeRange + alias Plausible.Stats.Query require Logger @@ -65,12 +65,11 @@ defmodule Plausible.Google.API do {:ok, access_token} <- maybe_refresh_token(site.google_auth), {:ok, gsc_filters} <- SearchConsole.Filters.transform(site.google_auth.property, query.filters, search), - date_range = DateTimeRange.to_date_range(query.date_range), {:ok, stats} <- HTTP.list_stats( access_token, site.google_auth.property, - date_range, + Query.date_range(query), pagination, gsc_filters ) do diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 76f08c37eb2b..be636e1231d0 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -116,6 +116,8 @@ defmodule Plausible.Stats.Breakdown do from e in subquery(timed_page_transitions_q), group_by: e.pathname + date_range = Query.date_range(query) + timed_pages_q = if query.include_imported do # Imported page views have pre-calculated values @@ -123,9 +125,7 @@ defmodule Plausible.Stats.Breakdown do from i in "imported_pages", group_by: i.page, where: i.site_id == ^site.id, - where: - i.date >= ^DateTime.to_naive(query.date_range.first) and - i.date <= ^DateTime.to_naive(query.date_range.last), + where: i.date >= ^date_range.first and i.date <= ^date_range.last, where: i.page in ^pages, select: %{ page: i.page, diff --git a/lib/plausible/stats/comparisons.ex b/lib/plausible/stats/comparisons.ex index 3f5c1ab7c32c..22caaa5667bd 100644 --- a/lib/plausible/stats/comparisons.ex +++ b/lib/plausible/stats/comparisons.ex @@ -64,15 +64,17 @@ defmodule Plausible.Stats.Comparisons do |> Keyword.put_new(:now, DateTime.now!(site.timezone)) |> Keyword.put_new(:match_day_of_week?, false) - source_date_range = DateTimeRange.to_date_range(source_query.date_range) + source_date_range = Query.date_range(source_query) with :ok <- validate_mode(source_query, mode), {:ok, comparison_date_range} <- get_comparison_date_range(source_date_range, mode, opts) do - %Date.Range{first: first, last: last} = comparison_date_range + new_range = + DateTimeRange.new!(comparison_date_range.first, comparison_date_range.last, site.timezone) + |> DateTimeRange.to_timezone("Etc/UTC") comparison_query = source_query - |> Query.set(date_range: DateTimeRange.new!(first, last, site.timezone)) + |> Query.set(utc_time_range: new_range) |> maybe_include_imported(source_query) {:ok, comparison_query} diff --git a/lib/plausible/stats/datetime_range.ex b/lib/plausible/stats/datetime_range.ex index e8e787a71c21..e6df6cb8d1c1 100644 --- a/lib/plausible/stats/datetime_range.ex +++ b/lib/plausible/stats/datetime_range.ex @@ -45,7 +45,16 @@ defmodule Plausible.Stats.DateTimeRange do %__MODULE__{first: first, last: last} end - def to_date_range(%__MODULE__{first: first, last: last}) do + def to_timezone(%__MODULE__{first: first, last: last}, timezone) do + first = DateTime.shift_zone!(first, timezone) + last = DateTime.shift_zone!(last, timezone) + + %__MODULE__{first: first, last: last} + end + + def to_date_range(datetime_range, timezone) do + %__MODULE__{first: first, last: last} = to_timezone(datetime_range, timezone) + first = DateTime.to_date(first) last = DateTime.to_date(last) diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.ex index a0878122bc6d..d800a9b1adac 100644 --- a/lib/plausible/stats/filters/query_parser.ex +++ b/lib/plausible/stats/filters/query_parser.ex @@ -18,7 +18,9 @@ defmodule Plausible.Stats.Filters.QueryParser do with :ok <- JSONSchema.validate(schema_type, params), {:ok, date} <- parse_date(site, Map.get(params, "date"), date), - {:ok, date_range} <- parse_date_range(site, Map.get(params, "date_range"), date, now), + {:ok, raw_time_range} <- + parse_time_range(site, Map.get(params, "date_range"), date, now), + utc_time_range = raw_time_range |> DateTimeRange.to_timezone("Etc/UTC"), {:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])), {:ok, filters} <- parse_filters(Map.get(params, "filters", [])), {:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])), @@ -28,10 +30,10 @@ defmodule Plausible.Stats.Filters.QueryParser do query = %{ metrics: metrics, filters: filters, - date_range: date_range, + utc_time_range: utc_time_range, dimensions: dimensions, order_by: order_by, - timezone: date_range.first.time_zone, + timezone: site.timezone, preloaded_goals: preloaded_goals, include: include }, @@ -151,7 +153,7 @@ defmodule Plausible.Stats.Filters.QueryParser do {:ok, date} end - defp parse_date_range(_site, date_range, _date, now) when date_range in ["realtime", "30m"] do + defp parse_time_range(_site, date_range, _date, now) when date_range in ["realtime", "30m"] do duration_minutes = case date_range do "realtime" -> 5 @@ -164,27 +166,27 @@ defmodule Plausible.Stats.Filters.QueryParser do {:ok, DateTimeRange.new!(first_datetime, last_datetime)} end - defp parse_date_range(site, "day", date, _now) do + defp parse_time_range(site, "day", date, _now) do {:ok, DateTimeRange.new!(date, date, site.timezone)} end - defp parse_date_range(site, "7d", date, _now) do + defp parse_time_range(site, "7d", date, _now) do first = date |> Date.add(-6) {:ok, DateTimeRange.new!(first, date, site.timezone)} end - defp parse_date_range(site, "30d", date, _now) do + defp parse_time_range(site, "30d", date, _now) do first = date |> Date.add(-30) {:ok, DateTimeRange.new!(first, date, site.timezone)} end - defp parse_date_range(site, "month", date, _now) do + defp parse_time_range(site, "month", date, _now) do last = date |> Date.end_of_month() first = last |> Date.beginning_of_month() {:ok, DateTimeRange.new!(first, last, site.timezone)} end - defp parse_date_range(site, "6mo", date, _now) do + defp parse_time_range(site, "6mo", date, _now) do last = date |> Date.end_of_month() first = @@ -195,7 +197,7 @@ defmodule Plausible.Stats.Filters.QueryParser do {:ok, DateTimeRange.new!(first, last, site.timezone)} end - defp parse_date_range(site, "12mo", date, _now) do + defp parse_time_range(site, "12mo", date, _now) do last = date |> Date.end_of_month() first = @@ -206,19 +208,19 @@ defmodule Plausible.Stats.Filters.QueryParser do {:ok, DateTimeRange.new!(first, last, site.timezone)} end - defp parse_date_range(site, "year", date, _now) do + defp parse_time_range(site, "year", date, _now) do last = date |> Timex.end_of_year() first = last |> Timex.beginning_of_year() {:ok, DateTimeRange.new!(first, last, site.timezone)} end - defp parse_date_range(site, "all", date, _now) do + defp parse_time_range(site, "all", date, _now) do start_date = Plausible.Sites.stats_start_date(site) || date {:ok, DateTimeRange.new!(start_date, date, site.timezone)} end - defp parse_date_range(site, [from, to], _date, _now) + defp parse_time_range(site, [from, to], _date, _now) when is_binary(from) and is_binary(to) do case date_range_from_date_strings(site, from, to) do {:ok, date_range} -> {:ok, date_range} @@ -226,7 +228,7 @@ defmodule Plausible.Stats.Filters.QueryParser do end end - defp parse_date_range(_site, unknown, _date, _now), + defp parse_time_range(_site, unknown, _date, _now), do: {:error, "Invalid date_range '#{i(unknown)}'."} defp date_range_from_date_strings(site, from, to) do @@ -237,22 +239,14 @@ defmodule Plausible.Stats.Filters.QueryParser do end defp date_range_from_timestamps(from, to) do - with {:ok, from_datetime} <- datetime_from_timestamp(from), - {:ok, to_datetime} <- datetime_from_timestamp(to), - true <- from_datetime.time_zone == to_datetime.time_zone do + with {:ok, from_datetime, _offset} <- DateTime.from_iso8601(from), + {:ok, to_datetime, _offset} <- DateTime.from_iso8601(to) do {:ok, DateTimeRange.new!(from_datetime, to_datetime)} else _ -> {:error, "Invalid date_range '#{i([from, to])}'."} end end - defp datetime_from_timestamp(timestamp_string) do - with [timestamp, timezone] <- String.split(timestamp_string), - {:ok, naive_datetime} <- NaiveDateTime.from_iso8601(timestamp) do - DateTime.from_naive(naive_datetime, timezone) - end - end - defp today(site), do: DateTime.now!(site.timezone) |> DateTime.to_date() defp parse_dimensions(dimensions) when is_list(dimensions) do diff --git a/lib/plausible/stats/goal_suggestions.ex b/lib/plausible/stats/goal_suggestions.ex index d3192bfb81c1..93ee2cd57e25 100644 --- a/lib/plausible/stats/goal_suggestions.ex +++ b/lib/plausible/stats/goal_suggestions.ex @@ -57,13 +57,13 @@ defmodule Plausible.Stats.GoalSuggestions do ) |> maybe_set_limit(limit) + date_range = Query.date_range(query) + imported_q = from(i in "imported_custom_events", where: i.site_id == ^site.id, where: i.import_id in ^site.complete_import_ids, - where: - i.date >= ^DateTime.to_naive(query.date_range.first) and - i.date <= ^DateTime.to_naive(query.date_range.last), + where: i.date >= ^date_range.first and i.date <= ^date_range.last, where: i.visitors > 0, where: fragment("? ilike ?", i.name, ^matches), where: fragment("trim(?)", i.name) != "", diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.ex index 4a8744b5d56d..49fea32c9fb4 100644 --- a/lib/plausible/stats/imported/base.ex +++ b/lib/plausible/stats/imported/base.ex @@ -6,7 +6,7 @@ defmodule Plausible.Stats.Imported.Base do import Ecto.Query alias Plausible.Imported - alias Plausible.Stats.{Query, DateTimeRange} + alias Plausible.Stats.Query import Plausible.Stats.Filters, only: [dimensions_used_in_filters: 1] @@ -59,7 +59,8 @@ defmodule Plausible.Stats.Imported.Base do def query_imported(table, site, query) do import_ids = site.complete_import_ids - %{first: date_from, last: date_to} = DateTimeRange.to_date_range(query.date_range) + # Assumption: dates in imported table are in user-local timezone. + %{first: date_from, last: date_to} = Query.date_range(query) from(i in table, where: i.site_id == ^site.id, diff --git a/lib/plausible/stats/imported/sql/expression.ex b/lib/plausible/stats/imported/sql/expression.ex index 776796950e69..4ef755f3da2e 100644 --- a/lib/plausible/stats/imported/sql/expression.ex +++ b/lib/plausible/stats/imported/sql/expression.ex @@ -9,6 +9,8 @@ defmodule Plausible.Stats.Imported.SQL.Expression do import Plausible.Stats.Util, only: [shortname: 2] import Ecto.Query + alias Plausible.Stats.Query + @no_ref "Direct / None" @not_set "(not set)" @none "(none)" @@ -292,8 +294,10 @@ defmodule Plausible.Stats.Imported.SQL.Expression do end defp select_group_fields(q, "time:week", key, query) do + date_range = Query.date_range(query) + select_merge_as(q, [i], %{ - key => weekstart_not_before(i.date, ^DateTime.to_naive(query.date_range.first)) + key => weekstart_not_before(i.date, ^date_range.first) }) end diff --git a/lib/plausible/stats/interval.ex b/lib/plausible/stats/interval.ex index fc3324d4c58a..8f053f1e1c83 100644 --- a/lib/plausible/stats/interval.ex +++ b/lib/plausible/stats/interval.ex @@ -40,9 +40,7 @@ defmodule Plausible.Stats.Interval do @doc """ Returns the suggested interval for the given `DateTimeRange` struct. """ - def default_for_date_range(%DateTimeRange{} = date_range) do - %Date.Range{first: first, last: last} = DateTimeRange.to_date_range(date_range) - + def default_for_date_range(%DateTimeRange{first: first, last: last}) do cond do Timex.diff(last, first, :months) > 0 -> "month" diff --git a/lib/plausible/stats/legacy/legacy_query_builder.ex b/lib/plausible/stats/legacy/legacy_query_builder.ex index 7165e3d18f8e..065da1a3c80f 100644 --- a/lib/plausible/stats/legacy/legacy_query_builder.ex +++ b/lib/plausible/stats/legacy/legacy_query_builder.ex @@ -8,14 +8,14 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do alias Plausible.Stats.{Filters, Interval, Query, DateTimeRange, Metrics} - def from(site, params, debug_metadata) do - now = DateTime.utc_now(:second) + def from(site, params, debug_metadata, now \\ nil) do + now = now || DateTime.utc_now(:second) query = Query |> struct!(now: now, debug_metadata: debug_metadata) |> put_period(site, params) - |> put_timezone() + |> put_timezone(site) |> put_dimensions(params) |> put_interval(params) |> put_parsed_filters(params) @@ -53,86 +53,109 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do first_datetime = DateTime.shift(now, minute: -duration_minutes) last_datetime = DateTime.shift(now, second: 5) - struct!(query, period: period, date_range: DateTimeRange.new!(first_datetime, last_datetime)) + datetime_range = + DateTimeRange.new!(first_datetime, last_datetime) |> DateTimeRange.to_timezone("Etc/UTC") + + struct!(query, period: period, utc_time_range: datetime_range) end defp put_period(query, site, %{"period" => "day"} = params) do - date = parse_single_date(site.timezone, params) - datetime_range = DateTimeRange.new!(date, date, site.timezone) + date = parse_single_date(query, params) + + datetime_range = + DateTimeRange.new!(date, date, site.timezone) |> DateTimeRange.to_timezone("Etc/UTC") - struct!(query, period: "day", date_range: datetime_range) + struct!(query, period: "day", utc_time_range: datetime_range) end defp put_period(query, site, %{"period" => "7d"} = params) do - end_date = parse_single_date(site.timezone, params) + end_date = parse_single_date(query, params) start_date = end_date |> Date.shift(day: -6) - datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone) - struct!(query, period: "7d", date_range: datetime_range) + datetime_range = + DateTimeRange.new!(start_date, end_date, site.timezone) + |> DateTimeRange.to_timezone("Etc/UTC") + + struct!(query, period: "7d", utc_time_range: datetime_range) end defp put_period(query, site, %{"period" => "30d"} = params) do - end_date = parse_single_date(site.timezone, params) + end_date = parse_single_date(query, params) start_date = end_date |> Date.shift(day: -30) - datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone) - struct!(query, period: "30d", date_range: datetime_range) + datetime_range = + DateTimeRange.new!(start_date, end_date, site.timezone) + |> DateTimeRange.to_timezone("Etc/UTC") + + struct!(query, period: "30d", utc_time_range: datetime_range) end defp put_period(query, site, %{"period" => "month"} = params) do - date = parse_single_date(site.timezone, params) + date = parse_single_date(query, params) start_date = Timex.beginning_of_month(date) end_date = Timex.end_of_month(date) - datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone) - struct!(query, period: "month", date_range: datetime_range) + datetime_range = + DateTimeRange.new!(start_date, end_date, site.timezone) + |> DateTimeRange.to_timezone("Etc/UTC") + + struct!(query, period: "month", utc_time_range: datetime_range) end defp put_period(query, site, %{"period" => "6mo"} = params) do end_date = - parse_single_date(site.timezone, params) + parse_single_date(query, params) |> Timex.end_of_month() start_date = Date.shift(end_date, month: -5) |> Timex.beginning_of_month() - datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone) + datetime_range = + DateTimeRange.new!(start_date, end_date, site.timezone) + |> DateTimeRange.to_timezone("Etc/UTC") - struct!(query, period: "6mo", date_range: datetime_range) + struct!(query, period: "6mo", utc_time_range: datetime_range) end defp put_period(query, site, %{"period" => "12mo"} = params) do end_date = - parse_single_date(site.timezone, params) + parse_single_date(query, params) |> Timex.end_of_month() start_date = Date.shift(end_date, month: -11) |> Timex.beginning_of_month() - datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone) + datetime_range = + DateTimeRange.new!(start_date, end_date, site.timezone) + |> DateTimeRange.to_timezone("Etc/UTC") - struct!(query, period: "12mo", date_range: datetime_range) + struct!(query, period: "12mo", utc_time_range: datetime_range) end defp put_period(query, site, %{"period" => "year"} = params) do end_date = - parse_single_date(site.timezone, params) + parse_single_date(query, params) |> Timex.end_of_year() start_date = Timex.beginning_of_year(end_date) - datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone) - struct!(query, period: "year", date_range: datetime_range) + datetime_range = + DateTimeRange.new!(start_date, end_date, site.timezone) + |> DateTimeRange.to_timezone("Etc/UTC") + + struct!(query, period: "year", utc_time_range: datetime_range) end defp put_period(query, site, %{"period" => "all"}) do - today = today(site.timezone) + today = today(query) start_date = Plausible.Sites.stats_start_date(site) || today - datetime_range = DateTimeRange.new!(start_date, today, site.timezone) - struct!(query, period: "all", date_range: datetime_range) + datetime_range = + DateTimeRange.new!(start_date, today, site.timezone) |> DateTimeRange.to_timezone("Etc/UTC") + + struct!(query, period: "all", utc_time_range: datetime_range) end defp put_period(query, site, %{"period" => "custom", "from" => from, "to" => to} = params) do @@ -148,17 +171,20 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do [from, to] = String.split(date, ",") from_date = Date.from_iso8601!(String.trim(from)) to_date = Date.from_iso8601!(String.trim(to)) - datetime_range = DateTimeRange.new!(from_date, to_date, site.timezone) - struct!(query, period: "custom", date_range: datetime_range) + datetime_range = + DateTimeRange.new!(from_date, to_date, site.timezone) + |> DateTimeRange.to_timezone("Etc/UTC") + + struct!(query, period: "custom", utc_time_range: datetime_range) end defp put_period(query, site, params) do put_period(query, site, Map.merge(params, %{"period" => "30d"})) end - defp put_timezone(query) do - struct!(query, timezone: query.date_range.first.time_zone) + defp put_timezone(query, site) do + struct!(query, timezone: site.timezone) end defp put_dimensions(query, params) do @@ -225,7 +251,7 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do end defp put_interval(%{:period => "all"} = query, params) do - interval = Map.get(params, "interval", Interval.default_for_date_range(query.date_range)) + interval = Map.get(params, "interval", Interval.default_for_date_range(query.utc_time_range)) struct!(query, interval: interval) end @@ -238,15 +264,15 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do struct!(query, filters: Filters.parse(params["filters"])) end - defp today(tz) do - DateTime.now!(tz) |> Timex.to_date() + defp today(query) do + query.now |> Timex.to_date() end - defp parse_single_date(tz, params) do + defp parse_single_date(query, params) do case params["date"] do - "today" -> DateTime.now!(tz) |> Timex.to_date() + "today" -> query.now |> Timex.to_date() date when is_binary(date) -> Date.from_iso8601!(date) - _ -> today(tz) + _ -> today(query) end end end diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index ee2d6556f4e0..682e854c105b 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -1,7 +1,7 @@ defmodule Plausible.Stats.Query do use Plausible - defstruct date_range: nil, + defstruct utc_time_range: nil, interval: nil, period: nil, dimensions: [], @@ -44,8 +44,8 @@ defmodule Plausible.Stats.Query do @doc """ Builds query from old-style params. New code should prefer Query.build """ - def from(site, params, debug_metadata \\ %{}) do - Legacy.QueryBuilder.from(site, params, debug_metadata) + def from(site, params, debug_metadata \\ %{}, now \\ nil) do + Legacy.QueryBuilder.from(site, params, debug_metadata, now) end def put_experimental_reduced_joins(query, site, params) do @@ -60,6 +60,10 @@ defmodule Plausible.Stats.Query do end end + def date_range(query) do + Plausible.Stats.DateTimeRange.to_date_range(query.utc_time_range, query.timezone) + end + def set(query, keywords) do new_query = struct!(query, keywords) @@ -141,7 +145,7 @@ defmodule Plausible.Stats.Query do "time:minute" in query.dimensions or "time:hour" in query.dimensions -> {:error, :unsupported_interval} - Date.after?(query.date_range.first, query.latest_import_end_date) -> + Date.after?(date_range(query).first, query.latest_import_end_date) -> {:error, :out_of_range} not Imported.schema_supports_query?(query) -> diff --git a/lib/plausible/stats/query_optimizer.ex b/lib/plausible/stats/query_optimizer.ex index c93958503e5c..7d43f7b2afe4 100644 --- a/lib/plausible/stats/query_optimizer.ex +++ b/lib/plausible/stats/query_optimizer.ex @@ -61,7 +61,7 @@ defmodule Plausible.Stats.QueryOptimizer do defp update_group_by_time( %Query{ - date_range: %DateTimeRange{first: first, last: last} + utc_time_range: %DateTimeRange{first: first, last: last} } = query ) do dimensions = diff --git a/lib/plausible/stats/query_result.ex b/lib/plausible/stats/query_result.ex index 66835d646c8c..c332c88a0a6d 100644 --- a/lib/plausible/stats/query_result.ex +++ b/lib/plausible/stats/query_result.ex @@ -32,8 +32,8 @@ defmodule Plausible.Stats.QueryResult do site_id: site.domain, metrics: query.metrics, date_range: [ - to_iso_8601_with_timezone(query.date_range.first), - to_iso_8601_with_timezone(query.date_range.last) + to_iso8601(query.utc_time_range.first, query.timezone), + to_iso8601(query.utc_time_range.last, query.timezone) ], filters: query.filters, dimensions: query.dimensions, @@ -88,13 +88,10 @@ defmodule Plausible.Stats.QueryResult do |> Enum.into(%{}) end - defp to_iso_8601_with_timezone(%DateTime{time_zone: timezone} = datetime) do - naive_iso8601 = - datetime - |> DateTime.to_naive() - |> NaiveDateTime.to_iso8601() - - naive_iso8601 <> " " <> timezone + defp to_iso8601(datetime, timezone) do + datetime + |> DateTime.shift_zone!(timezone) + |> DateTime.to_iso8601(:extended) end end diff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.ex index e5e8fb447293..4a051e0c6d76 100644 --- a/lib/plausible/stats/sql/expression.ex +++ b/lib/plausible/stats/sql/expression.ex @@ -12,7 +12,7 @@ defmodule Plausible.Stats.SQL.Expression do import Ecto.Query - alias Plausible.Stats.{Filters, SQL} + alias Plausible.Stats.{Query, Filters, SQL} @no_ref "Direct / None" @not_set "(not set)" @@ -46,11 +46,13 @@ defmodule Plausible.Stats.SQL.Expression do end def select_dimension(q, key, "time:week", _table, query) do + date_range = Query.date_range(query) + select_merge_as(q, [t], %{ key => weekstart_not_before( to_timezone(t.timestamp, ^query.timezone), - ^DateTime.to_naive(query.date_range.first) + ^date_range.first ) }) end diff --git a/lib/plausible/stats/time.ex b/lib/plausible/stats/time.ex index 4b9b4b6a53dd..7be4866e375f 100644 --- a/lib/plausible/stats/time.ex +++ b/lib/plausible/stats/time.ex @@ -5,16 +5,13 @@ defmodule Plausible.Stats.Time do alias Plausible.Stats.{Query, DateTimeRange} - def utc_boundaries(%Query{date_range: date_range}, site) do - %DateTimeRange{first: first, last: last} = date_range - + def utc_boundaries(%Query{utc_time_range: time_range}, site) do first = - first - |> DateTime.shift_zone!("Etc/UTC") + time_range.first |> DateTime.to_naive() |> beginning_of_time(site.native_stats_start_at) - last = DateTime.shift_zone!(last, "Etc/UTC") |> DateTime.to_naive() + last = DateTime.to_naive(time_range.last) {first, last} end @@ -47,7 +44,7 @@ defmodule Plausible.Stats.Time do end defp time_labels_for_dimension("time:month", query) do - date_range = DateTimeRange.to_date_range(query.date_range) + date_range = Query.date_range(query) n_buckets = Timex.diff( @@ -65,7 +62,7 @@ defmodule Plausible.Stats.Time do end defp time_labels_for_dimension("time:week", query) do - date_range = DateTimeRange.to_date_range(query.date_range) + date_range = Query.date_range(query) n_buckets = Timex.diff( @@ -77,23 +74,25 @@ defmodule Plausible.Stats.Time do Enum.map(0..n_buckets, fn shift -> date_range.first |> Date.shift(week: shift) - |> date_or_weekstart(query) + |> date_or_weekstart(date_range) |> format_datetime() end) end defp time_labels_for_dimension("time:day", query) do - query.date_range - |> DateTimeRange.to_date_range() + Query.date_range(query) |> Enum.into([]) |> Enum.map(&format_datetime/1) end defp time_labels_for_dimension("time:hour", query) do - n_buckets = DateTime.diff(query.date_range.last, query.date_range.first, :hour) + time_range = query.utc_time_range |> DateTimeRange.to_timezone(query.timezone) + + from_timestamp = time_range.first |> Map.merge(%{minute: 0, second: 0}) + n_buckets = DateTime.diff(time_range.last, from_timestamp, :hour) Enum.map(0..n_buckets, fn step -> - query.date_range.first + from_timestamp |> DateTime.to_naive() |> NaiveDateTime.shift(hour: step) |> format_datetime() @@ -101,24 +100,23 @@ defmodule Plausible.Stats.Time do end defp time_labels_for_dimension("time:minute", query) do - first_datetime = Map.put(query.date_range.first, :second, 0) + time_range = query.utc_time_range |> DateTimeRange.to_timezone(query.timezone) + first_datetime = Map.put(time_range.first, :second, 0) first_datetime |> Stream.iterate(fn datetime -> DateTime.shift(datetime, minute: 1) end) |> Enum.take_while(fn datetime -> current_minute = Map.put(query.now, :second, 0) - DateTime.before?(datetime, query.date_range.last) && + DateTime.before?(datetime, time_range.last) && DateTime.before?(datetime, current_minute) end) |> Enum.map(&format_datetime/1) end - def date_or_weekstart(date, query) do + def date_or_weekstart(date, date_range) do weekstart = Date.beginning_of_week(date) - date_range = DateTimeRange.to_date_range(query.date_range) - if Enum.member?(date_range, weekstart) do weekstart else diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 689c6aa157c4..8fa9a895ebee 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -5,7 +5,7 @@ defmodule PlausibleWeb.Api.StatsController do use PlausibleWeb.Plugs.ErrorHandler alias Plausible.Stats - alias Plausible.Stats.{Query, Comparisons, Filters, Time, TableDecider, DateTimeRange} + alias Plausible.Stats.{Query, Comparisons, Filters, Time, TableDecider} alias Plausible.Stats.Filters.LegacyDashboardFilterParser alias PlausibleWeb.Api.Helpers, as: H @@ -164,13 +164,19 @@ defmodule PlausibleWeb.Api.StatsController do end end - defp build_full_intervals(%{interval: "week", date_range: date_range}, labels) do - date_range = DateTimeRange.to_date_range(date_range) + defp build_full_intervals( + %Query{interval: "week"} = query, + labels + ) do + date_range = Query.date_range(query) build_intervals(labels, date_range, &Timex.beginning_of_week/1, &Timex.end_of_week/1) end - defp build_full_intervals(%{interval: "month", date_range: date_range}, labels) do - date_range = DateTimeRange.to_date_range(date_range) + defp build_full_intervals( + %Query{interval: "month"} = query, + labels + ) do + date_range = Query.date_range(query) build_intervals(labels, date_range, &Timex.beginning_of_month/1, &Timex.end_of_month/1) end @@ -220,10 +226,10 @@ defmodule PlausibleWeb.Api.StatsController do with_imported_switch: with_imported_switch_info(query, comparison_query), includes_imported: includes_imported?(query, comparison_query), imports_exist: site.complete_import_ids != [], - comparing_from: comparison_query && DateTime.to_date(comparison_query.date_range.first), - comparing_to: comparison_query && DateTime.to_date(comparison_query.date_range.last), - from: DateTime.to_date(query.date_range.first), - to: DateTime.to_date(query.date_range.last) + comparing_from: comparison_query && Query.date_range(comparison_query).first, + comparing_to: comparison_query && Query.date_range(comparison_query).last, + from: Query.date_range(query).first, + to: Query.date_range(query).last }) end @@ -277,10 +283,12 @@ defmodule PlausibleWeb.Api.StatsController do Enum.find_index(dates, &(&1 == current_date)) "week" -> + date_range = Query.date_range(query) + current_date = DateTime.now!(site.timezone) |> Timex.to_date() - |> Time.date_or_weekstart(query) + |> Time.date_or_weekstart(date_range) |> Date.to_string() Enum.find_index(dates, &(&1 == current_date)) diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex index e5e568cc278f..dd1d52e450c2 100644 --- a/lib/plausible_web/controllers/stats_controller.ex +++ b/lib/plausible_web/controllers/stats_controller.ex @@ -110,8 +110,10 @@ defmodule PlausibleWeb.StatsController do site = Plausible.Repo.preload(conn.assigns.site, :owner) query = Query.from(site, params, debug_metadata(conn)) + date_range = Query.date_range(query) + filename = - ~c"Plausible export #{params["domain"]} #{Date.to_iso8601(query.date_range.first)} to #{Date.to_iso8601(query.date_range.last)} .zip" + ~c"Plausible export #{params["domain"]} #{Date.to_iso8601(date_range.first)} to #{Date.to_iso8601(date_range.last)} .zip" params = Map.merge(params, %{"limit" => "300", "csv" => "True", "detailed" => "True"}) limited_params = Map.merge(params, %{"limit" => "100"}) diff --git a/lib/plausible_web/mjml/templates/stats_report.mjml.eex b/lib/plausible_web/mjml/templates/stats_report.mjml.eex index 2818c93307da..51782cba66c7 100644 --- a/lib/plausible_web/mjml/templates/stats_report.mjml.eex +++ b/lib/plausible_web/mjml/templates/stats_report.mjml.eex @@ -22,7 +22,7 @@ <%= @site.domain %> - <%= @name %> Report (<%= Calendar.strftime(@query.date_range.last, "%-d %b %Y") %>) + <%= @name %> Report (<%= @date %>) diff --git a/lib/workers/send_email_report.ex b/lib/workers/send_email_report.ex index 85b31986ad6f..4494ea39506c 100644 --- a/lib/workers/send_email_report.ex +++ b/lib/workers/send_email_report.ex @@ -9,9 +9,11 @@ defmodule Plausible.Workers.SendEmailReport do if site && site.weekly_report do %{site: site} + |> put_last_week_query() + |> put_date_range() |> Map.put(:type, :weekly) |> Map.put(:name, "Weekly") - |> put_last_week_query() + |> put(:date, &Calendar.strftime(&1.date_range.last, "%-d %b %Y")) |> put_stats() |> send_report_for_all(site.weekly_report.recipients) else @@ -25,9 +27,11 @@ defmodule Plausible.Workers.SendEmailReport do if site && site.monthly_report do %{site: site} - |> Map.put(:type, :monthly) |> put_last_month_query() - |> put_monthly_report_name() + |> put_date_range() + |> Map.put(:type, :monthly) + |> put(:name, &Calendar.strftime(&1.date_range.first, "%B")) + |> put(:date, &Calendar.strftime(&1.date_range.last, "%-d %b %Y")) |> put_stats() |> send_report_for_all(site.monthly_report.recipients) else @@ -76,11 +80,15 @@ defmodule Plausible.Workers.SendEmailReport do Map.put(assigns, :query, query) end - defp put_monthly_report_name(%{query: query} = assigns) do - Map.put(assigns, :name, Calendar.strftime(query.date_range.first, "%B")) + defp put_date_range(%{query: query} = assigns) do + Map.put(assigns, :date_range, Query.date_range(query)) end defp put_stats(%{site: site, query: query} = assigns) do Map.put(assigns, :stats, Plausible.Stats.EmailReport.get(site, query)) end + + defp put(assigns, key, value_fn) do + Map.put(assigns, key, value_fn.(assigns)) + end end diff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.json index 9b80aac218ba..b6c062f3341f 100644 --- a/priv/json-schemas/query-api-schema.json +++ b/priv/json-schemas/query-api-schema.json @@ -95,15 +95,23 @@ "minItems": 2, "maxItems": 2, "items": { - "type": "string", - "pattern": "^\\d{4}-\\d{2}-\\d{2}(?:T\\d{2}:\\d{2}:\\d{2}\\s[A-Za-z/_]+)?$" + "oneOf": [ + { + "type": "string", + "format": "date" + }, + { + "type": "string", + "format": "date-time" + } + ] }, - "markdownDescription": "A list of two elements to determine the query date range. Both elements should have the same format - either `YYYY-MM-DD` or `YYYY-MM-DDThh:mm:ss `", + "markdownDescription": "A list of two ISO8601 dates or timestamps to determine the query date range.", "examples": [ ["2024-01-01", "2024-01-31"], [ - "2024-01-01T00:00:00 Europe/Tallinn", - "2024-01-01T12:00:00 Europe/Tallinn" + "2024-01-01T00:00:00+03:00", + "2024-01-02T12:00:00+03:00" ] ] } diff --git a/test/plausible/google/api_test.exs b/test/plausible/google/api_test.exs index ef6a96d93a17..4724d360a7d6 100644 --- a/test/plausible/google/api_test.exs +++ b/test/plausible/google/api_test.exs @@ -3,7 +3,7 @@ defmodule Plausible.Google.APITest do use Plausible.Test.Support.HTTPMocker alias Plausible.Google - alias Plausible.Stats.Query + alias Plausible.Stats.{DateTimeRange, Query} import ExUnit.CaptureLog import Mox @@ -111,7 +111,8 @@ defmodule Plausible.Google.APITest do end test "returns error when google auth not configured", %{site: site} do - query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])} + time_range = DateTimeRange.new!(~U[2022-01-01 00:00:00Z], ~U[2022-01-05 23:59:59Z]) + query = %Plausible.Stats.Query{utc_time_range: time_range} assert {:error, :google_property_not_configured} = Google.API.fetch_stats(site, query, 5, "") end diff --git a/test/plausible/stats/comparisons_test.exs b/test/plausible/stats/comparisons_test.exs index 7d2504c6a7dd..f2408dc173a3 100644 --- a/test/plausible/stats/comparisons_test.exs +++ b/test/plausible/stats/comparisons_test.exs @@ -11,11 +11,8 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now) - assert comparison.date_range.first == - DateTime.new!(~D[2023-02-27], ~T[00:00:00], site.timezone) - - assert comparison.date_range.last == - DateTime.new!(~D[2023-02-28], ~T[23:59:59], site.timezone) + assert comparison.utc_time_range.first == ~U[2023-02-27 00:00:00Z] + assert comparison.utc_time_range.last == ~U[2023-02-28 23:59:59Z] end test "shifts back this month period when it's the first day of the month and mode is previous_period" do @@ -25,11 +22,8 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now) - assert comparison.date_range.first == - DateTime.new!(~D[2023-02-28], ~T[00:00:00], site.timezone) - - assert comparison.date_range.last == - DateTime.new!(~D[2023-02-28], ~T[23:59:59], site.timezone) + assert comparison.utc_time_range.first == ~U[2023-02-28 00:00:00Z] + assert comparison.utc_time_range.last == ~U[2023-02-28 23:59:59Z] end test "matches the day of the week when nearest day is original query start date and mode is previous_period" do @@ -40,11 +34,19 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now, match_day_of_week?: true) - assert comparison.date_range.first == - DateTime.new!(~D[2023-02-22], ~T[00:00:00], site.timezone) + assert comparison.utc_time_range.first == ~U[2023-02-22 00:00:00Z] + assert comparison.utc_time_range.last == ~U[2023-02-23 23:59:59Z] + end + + test "custom time zone sets timezone to UTC" do + site = insert(:site, timezone: "US/Eastern") + query = Query.from(site, %{"period" => "month", "date" => "2023-03-02"}) + now = ~N[2023-03-02 14:00:00] - assert comparison.date_range.last == - DateTime.new!(~D[2023-02-23], ~T[23:59:59], site.timezone) + {:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now) + + assert comparison.utc_time_range.first == ~U[2023-02-27 05:00:00Z] + assert comparison.utc_time_range.last == ~U[2023-03-01 04:59:59Z] end end @@ -56,11 +58,8 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now) - assert comparison.date_range.first == - DateTime.new!(~D[2023-01-04], ~T[00:00:00], site.timezone) - - assert comparison.date_range.last == - DateTime.new!(~D[2023-01-31], ~T[23:59:59], site.timezone) + assert comparison.utc_time_range.first == ~U[2023-01-04 00:00:00Z] + assert comparison.utc_time_range.last == ~U[2023-01-31 23:59:59Z] end test "shifts back the full month when mode is year_over_year" do @@ -70,11 +69,8 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "year_over_year", now: now) - assert comparison.date_range.first == - DateTime.new!(~D[2022-02-01], ~T[00:00:00], site.timezone) - - assert comparison.date_range.last == - DateTime.new!(~D[2022-02-28], ~T[23:59:59], site.timezone) + assert comparison.utc_time_range.first == ~U[2022-02-01 00:00:00Z] + assert comparison.utc_time_range.last == ~U[2022-02-28 23:59:59Z] end test "shifts back whole month plus one day when mode is year_over_year and a leap year" do @@ -84,11 +80,8 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "year_over_year", now: now) - assert comparison.date_range.first == - DateTime.new!(~D[2019-02-01], ~T[00:00:00], site.timezone) - - assert comparison.date_range.last == - DateTime.new!(~D[2019-03-01], ~T[23:59:59], site.timezone) + assert comparison.utc_time_range.first == ~U[2019-02-01 00:00:00Z] + assert comparison.utc_time_range.last == ~U[2019-03-01 23:59:59Z] end test "matches the day of the week when mode is previous_period keeping the same day" do @@ -99,11 +92,8 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now, match_day_of_week?: true) - assert comparison.date_range.first == - DateTime.new!(~D[2023-01-04], ~T[00:00:00], site.timezone) - - assert comparison.date_range.last == - DateTime.new!(~D[2023-01-31], ~T[23:59:59], site.timezone) + assert comparison.utc_time_range.first == ~U[2023-01-04 00:00:00Z] + assert comparison.utc_time_range.last == ~U[2023-01-31 23:59:59Z] end test "matches the day of the week when mode is previous_period" do @@ -114,11 +104,8 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now, match_day_of_week?: true) - assert comparison.date_range.first == - DateTime.new!(~D[2022-12-04], ~T[00:00:00], site.timezone) - - assert comparison.date_range.last == - DateTime.new!(~D[2023-01-03], ~T[23:59:59], site.timezone) + assert comparison.utc_time_range.first == ~U[2022-12-04 00:00:00Z] + assert comparison.utc_time_range.last == ~U[2023-01-03 23:59:59Z] end end @@ -130,11 +117,8 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now) - assert comparison.date_range.first == - DateTime.new!(~D[2022-11-02], ~T[00:00:00], site.timezone) - - assert comparison.date_range.last == - DateTime.new!(~D[2022-12-31], ~T[23:59:59], site.timezone) + assert comparison.utc_time_range.first == ~U[2022-11-02 00:00:00Z] + assert comparison.utc_time_range.last == ~U[2022-12-31 23:59:59Z] end test "shifts back by the same number of days when mode is year_over_year" do @@ -144,11 +128,8 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "year_over_year", now: now) - assert comparison.date_range.first == - DateTime.new!(~D[2022-01-01], ~T[00:00:00], site.timezone) - - assert comparison.date_range.last == - DateTime.new!(~D[2022-03-01], ~T[23:59:59], site.timezone) + assert comparison.utc_time_range.first == ~U[2022-01-01 00:00:00Z] + assert comparison.utc_time_range.last == ~U[2022-03-01 23:59:59Z] end test "matches the day of the week when mode is year_over_year" do @@ -159,11 +140,8 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "year_over_year", now: now, match_day_of_week?: true) - assert comparison.date_range.first == - DateTime.new!(~D[2022-01-02], ~T[00:00:00], site.timezone) - - assert comparison.date_range.last == - DateTime.new!(~D[2022-03-02], ~T[23:59:59], site.timezone) + assert comparison.utc_time_range.first == ~U[2022-01-02 00:00:00Z] + assert comparison.utc_time_range.last == ~U[2022-03-02 23:59:59Z] end end @@ -174,11 +152,8 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "year_over_year") - assert comparison.date_range.first == - DateTime.new!(~D[2021-01-01], ~T[00:00:00], site.timezone) - - assert comparison.date_range.last == - DateTime.new!(~D[2021-12-31], ~T[23:59:59], site.timezone) + assert comparison.utc_time_range.first == ~U[2021-01-01 00:00:00Z] + assert comparison.utc_time_range.last == ~U[2021-12-31 23:59:59Z] end test "shifts back a whole year when mode is previous_period" do @@ -187,11 +162,8 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "previous_period") - assert comparison.date_range.first == - DateTime.new!(~D[2021-01-01], ~T[00:00:00], site.timezone) - - assert comparison.date_range.last == - DateTime.new!(~D[2021-12-31], ~T[23:59:59], site.timezone) + assert comparison.utc_time_range.first == ~U[2021-01-01 00:00:00Z] + assert comparison.utc_time_range.last == ~U[2021-12-31 23:59:59Z] end end @@ -202,11 +174,8 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "previous_period") - assert comparison.date_range.first == - DateTime.new!(~D[2022-12-25], ~T[00:00:00], site.timezone) - - assert comparison.date_range.last == - DateTime.new!(~D[2022-12-31], ~T[23:59:59], site.timezone) + assert comparison.utc_time_range.first == ~U[2022-12-25 00:00:00Z] + assert comparison.utc_time_range.last == ~U[2022-12-31 23:59:59Z] end test "shifts back to last year when mode is year_over_year" do @@ -215,11 +184,8 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "year_over_year") - assert comparison.date_range.first == - DateTime.new!(~D[2022-01-01], ~T[00:00:00], site.timezone) - - assert comparison.date_range.last == - DateTime.new!(~D[2022-01-07], ~T[23:59:59], site.timezone) + assert comparison.utc_time_range.first == ~U[2022-01-01 00:00:00Z] + assert comparison.utc_time_range.last == ~U[2022-01-07 23:59:59Z] end end @@ -231,11 +197,8 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "custom", from: "2022-05-25", to: "2022-05-30") - assert comparison.date_range.first == - DateTime.new!(~D[2022-05-25], ~T[00:00:00], site.timezone) - - assert comparison.date_range.last == - DateTime.new!(~D[2022-05-30], ~T[23:59:59], site.timezone) + assert comparison.utc_time_range.first == ~U[2022-05-25 00:00:00Z] + assert comparison.utc_time_range.last == ~U[2022-05-30 23:59:59Z] end test "validates from and to dates" do diff --git a/test/plausible/stats/query_optimizer_test.exs b/test/plausible/stats/query_optimizer_test.exs index 259d54028773..dc9e9a7d1b3f 100644 --- a/test/plausible/stats/query_optimizer_test.exs +++ b/test/plausible/stats/query_optimizer_test.exs @@ -24,7 +24,7 @@ defmodule Plausible.Stats.QueryOptimizerTest do test "adds time and first metric to order_by if order_by not specified" do assert perform(%{ - date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-31], "UTC"), + utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-31], "UTC"), metrics: [:pageviews, :visitors], dimensions: ["time", "event:page"] }).order_by == @@ -35,14 +35,14 @@ defmodule Plausible.Stats.QueryOptimizerTest do describe "update_group_by_time" do test "does nothing if `time` dimension not passed" do assert perform(%{ - date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-04], "UTC"), + utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-04], "UTC"), dimensions: ["time:month"] }).dimensions == ["time:month"] end test "updating time dimension" do assert perform(%{ - date_range: + utc_time_range: DateTimeRange.new!( DateTime.new!(~D[2022-01-01], ~T[00:00:00], "UTC"), DateTime.new!(~D[2022-01-01], ~T[05:00:00], "UTC") @@ -51,57 +51,57 @@ defmodule Plausible.Stats.QueryOptimizerTest do }).dimensions == ["time:hour"] assert perform(%{ - date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-01], "UTC"), + utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-01], "UTC"), dimensions: ["time"] }).dimensions == ["time:hour"] assert perform(%{ - date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-02], "UTC"), + utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-02], "UTC"), dimensions: ["time"] }).dimensions == ["time:hour"] assert perform(%{ - date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-03], "UTC"), + utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-03], "UTC"), dimensions: ["time"] }).dimensions == ["time:day"] assert perform(%{ - date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-10], "UTC"), + utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-10], "UTC"), dimensions: ["time"] }).dimensions == ["time:day"] assert perform(%{ - date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-16], "UTC"), + utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-16], "UTC"), dimensions: ["time"] }).dimensions == ["time:day"] assert perform(%{ - date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-02-16], "UTC"), + utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-02-16], "UTC"), dimensions: ["time"] }).dimensions == ["time:week"] assert perform(%{ - date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-03-16], "UTC"), + utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-03-16], "UTC"), dimensions: ["time"] }).dimensions == ["time:week"] assert perform(%{ - date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-03-16], "UTC"), + utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-03-16], "UTC"), dimensions: ["time"] }).dimensions == ["time:week"] assert perform(%{ - date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2023-11-16], "UTC"), + utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2023-11-16], "UTC"), dimensions: ["time"] }).dimensions == ["time:month"] assert perform(%{ - date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2024-01-16], "UTC"), + utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2024-01-16], "UTC"), dimensions: ["time"] }).dimensions == ["time:month"] assert perform(%{ - date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2026-01-01], "UTC"), + utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2026-01-01], "UTC"), dimensions: ["time"] }).dimensions == ["time:month"] end @@ -110,7 +110,7 @@ defmodule Plausible.Stats.QueryOptimizerTest do describe "update_time_in_order_by" do test "updates explicit time dimension in order_by" do assert perform(%{ - date_range: + utc_time_range: DateTimeRange.new!( DateTime.new!(~D[2022-01-01], ~T[00:00:00], "UTC"), DateTime.new!(~D[2022-01-01], ~T[05:00:00], "UTC") @@ -124,7 +124,7 @@ defmodule Plausible.Stats.QueryOptimizerTest do describe "extend_hostname_filters_to_visit" do test "updates filters it filtering by event:hostname and visit:referrer and visit:exit_page dimensions" do assert perform(%{ - date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]), + utc_time_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]), filters: [ [:is, "event:hostname", ["example.com"]], [:matches_wildcard, "event:hostname", ["*.com"]] @@ -142,7 +142,7 @@ defmodule Plausible.Stats.QueryOptimizerTest do test "does not update filters if not needed" do assert perform(%{ - date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]), + utc_time_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]), filters: [ [:is, "event:hostname", ["example.com"]] ], diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exs index 14f4e5ec8d13..8163ff810770 100644 --- a/test/plausible/stats/query_parser_test.exs +++ b/test/plausible/stats/query_parser_test.exs @@ -7,42 +7,42 @@ defmodule Plausible.Stats.Filters.QueryParserTest do setup [:create_user, :create_new_site] - @now DateTime.new!(~D[2021-05-05], ~T[12:30:00], "UTC") + @now DateTime.new!(~D[2021-05-05], ~T[12:30:00], "Etc/UTC") @date_range_realtime %DateTimeRange{ - first: DateTime.new!(~D[2021-05-05], ~T[12:25:00], "UTC"), - last: DateTime.new!(~D[2021-05-05], ~T[12:30:05], "UTC") + first: DateTime.new!(~D[2021-05-05], ~T[12:25:00], "Etc/UTC"), + last: DateTime.new!(~D[2021-05-05], ~T[12:30:05], "Etc/UTC") } @date_range_30m %DateTimeRange{ - first: DateTime.new!(~D[2021-05-05], ~T[12:00:00], "UTC"), - last: DateTime.new!(~D[2021-05-05], ~T[12:30:05], "UTC") + first: DateTime.new!(~D[2021-05-05], ~T[12:00:00], "Etc/UTC"), + last: DateTime.new!(~D[2021-05-05], ~T[12:30:05], "Etc/UTC") } @date_range_day %DateTimeRange{ - first: DateTime.new!(~D[2021-05-05], ~T[00:00:00], "UTC"), - last: DateTime.new!(~D[2021-05-05], ~T[23:59:59], "UTC") + first: DateTime.new!(~D[2021-05-05], ~T[00:00:00], "Etc/UTC"), + last: DateTime.new!(~D[2021-05-05], ~T[23:59:59], "Etc/UTC") } @date_range_7d %DateTimeRange{ - first: DateTime.new!(~D[2021-04-29], ~T[00:00:00], "UTC"), - last: DateTime.new!(~D[2021-05-05], ~T[23:59:59], "UTC") + first: DateTime.new!(~D[2021-04-29], ~T[00:00:00], "Etc/UTC"), + last: DateTime.new!(~D[2021-05-05], ~T[23:59:59], "Etc/UTC") } @date_range_30d %DateTimeRange{ - first: DateTime.new!(~D[2021-04-05], ~T[00:00:00], "UTC"), - last: DateTime.new!(~D[2021-05-05], ~T[23:59:59], "UTC") + first: DateTime.new!(~D[2021-04-05], ~T[00:00:00], "Etc/UTC"), + last: DateTime.new!(~D[2021-05-05], ~T[23:59:59], "Etc/UTC") } @date_range_month %DateTimeRange{ - first: DateTime.new!(~D[2021-05-01], ~T[00:00:00], "UTC"), - last: DateTime.new!(~D[2021-05-31], ~T[23:59:59], "UTC") + first: DateTime.new!(~D[2021-05-01], ~T[00:00:00], "Etc/UTC"), + last: DateTime.new!(~D[2021-05-31], ~T[23:59:59], "Etc/UTC") } @date_range_6mo %DateTimeRange{ - first: DateTime.new!(~D[2020-12-01], ~T[00:00:00], "UTC"), - last: DateTime.new!(~D[2021-05-31], ~T[23:59:59], "UTC") + first: DateTime.new!(~D[2020-12-01], ~T[00:00:00], "Etc/UTC"), + last: DateTime.new!(~D[2021-05-31], ~T[23:59:59], "Etc/UTC") } @date_range_year %DateTimeRange{ - first: DateTime.new!(~D[2021-01-01], ~T[00:00:00], "UTC"), - last: DateTime.new!(~D[2021-12-31], ~T[23:59:59], "UTC") + first: DateTime.new!(~D[2021-01-01], ~T[00:00:00], "Etc/UTC"), + last: DateTime.new!(~D[2021-12-31], ~T[23:59:59], "Etc/UTC") } @date_range_12mo %DateTimeRange{ - first: DateTime.new!(~D[2020-06-01], ~T[00:00:00], "UTC"), - last: DateTime.new!(~D[2021-05-31], ~T[23:59:59], "UTC") + first: DateTime.new!(~D[2020-06-01], ~T[00:00:00], "Etc/UTC"), + last: DateTime.new!(~D[2021-05-31], ~T[23:59:59], "Etc/UTC") } def check_success(params, site, expected_result, schema_type \\ :public) do @@ -55,7 +55,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do assert message == expected_error_message end - def check_date_range(date_params, site, expected_fields, schema_type \\ :public) do + def check_date_range(date_params, site, expected_date_range, schema_type \\ :public) do params = %{"site_id" => site.domain, "metrics" => ["visitors", "events"]} |> Map.merge(date_params) @@ -63,11 +63,11 @@ defmodule Plausible.Stats.Filters.QueryParserTest do expected_parsed = %{ metrics: [:visitors, :events], - date_range: expected_fields.date_range, + utc_time_range: expected_date_range, filters: [], dimensions: [], order_by: nil, - timezone: Map.get(expected_fields, :timezone, site.timezone), + timezone: site.timezone, include: %{imports: false, time_labels: false}, preloaded_goals: [] } @@ -85,7 +85,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do %{"site_id" => site.domain, "metrics" => ["visitors", "events"], "date_range" => "all"} |> check_success(site, %{ metrics: [:visitors, :events], - date_range: @date_range_day, + utc_time_range: @date_range_day, filters: [], dimensions: [], order_by: nil, @@ -126,7 +126,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do :bounce_rate, :visit_duration ], - date_range: @date_range_day, + utc_time_range: @date_range_day, filters: [], dimensions: [], order_by: nil, @@ -190,7 +190,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do site, %{ metrics: [:visitors], - date_range: @date_range_day, + utc_time_range: @date_range_day, filters: [ [unquote(operation), "event:name", ["foo"]] ], @@ -315,7 +315,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:visitors], - date_range: @date_range_day, + utc_time_range: @date_range_day, filters: [ [:is, "event:props:foobar", ["value"]] ], @@ -340,7 +340,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:visitors], - date_range: @date_range_day, + utc_time_range: @date_range_day, filters: [ [:is, "event:#{unquote(dimension)}", ["foo"]] ], @@ -366,7 +366,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:visitors], - date_range: @date_range_day, + utc_time_range: @date_range_day, filters: [ [:is, "visit:#{unquote(dimension)}", ["ab"]] ], @@ -432,7 +432,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:visitors], - date_range: @date_range_day, + utc_time_range: @date_range_day, filters: [ [:is, "visit:city", [123, 456]] ], @@ -451,7 +451,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:visitors], - date_range: @date_range_day, + utc_time_range: @date_range_day, filters: [ [:is, "visit:city", ["123", "456"]] ], @@ -499,7 +499,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:visitors], - date_range: @date_range_day, + utc_time_range: @date_range_day, filters: [ [ :or, @@ -558,7 +558,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:visitors], - date_range: @date_range_day, + utc_time_range: @date_range_day, filters: [ [:is, "event:hostname", ["a.plausible.io"]] ], @@ -595,7 +595,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:visitors], - date_range: @date_range_day, + utc_time_range: @date_range_day, filters: [], dimensions: ["time"], order_by: nil, @@ -645,7 +645,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do assert %{ metrics: [:visitors], - date_range: @date_range_day, + utc_time_range: @date_range_day, filters: [ [:is, "event:goal", ["Signup", "Visit /thank-you"]] ], @@ -729,42 +729,22 @@ defmodule Plausible.Stats.Filters.QueryParserTest do describe "date range validation" do test "parsing shortcut options", %{site: site} do - check_date_range(%{"date_range" => "day"}, site, %{date_range: @date_range_day}) - check_date_range(%{"date_range" => "7d"}, site, %{date_range: @date_range_7d}) - check_date_range(%{"date_range" => "30d"}, site, %{date_range: @date_range_30d}) - check_date_range(%{"date_range" => "month"}, site, %{date_range: @date_range_month}) - check_date_range(%{"date_range" => "6mo"}, site, %{date_range: @date_range_6mo}) - check_date_range(%{"date_range" => "12mo"}, site, %{date_range: @date_range_12mo}) - check_date_range(%{"date_range" => "year"}, site, %{date_range: @date_range_year}) + check_date_range(%{"date_range" => "day"}, site, @date_range_day) + check_date_range(%{"date_range" => "7d"}, site, @date_range_7d) + check_date_range(%{"date_range" => "30d"}, site, @date_range_30d) + check_date_range(%{"date_range" => "month"}, site, @date_range_month) + check_date_range(%{"date_range" => "6mo"}, site, @date_range_6mo) + check_date_range(%{"date_range" => "12mo"}, site, @date_range_12mo) + check_date_range(%{"date_range" => "year"}, site, @date_range_year) end test "30m and realtime are available in internal API", %{site: site} do - check_date_range(%{"date_range" => "30m"}, site, %{date_range: @date_range_30m}, :internal) - - check_date_range( - %{"date_range" => "realtime"}, - site, - %{date_range: @date_range_realtime}, - :internal - ) - end - - test "timezone is UTC instead of site.timezone for realtime and 30m periods", %{ - site: site - } do - site = struct!(site, timezone: "Europe/Tallinn") - - check_date_range( - %{"date_range" => "30m"}, - site, - %{date_range: @date_range_30m, timezone: "UTC"}, - :internal - ) + check_date_range(%{"date_range" => "30m"}, site, @date_range_30m, :internal) check_date_range( %{"date_range" => "realtime"}, site, - %{date_range: @date_range_realtime, timezone: "UTC"}, + @date_range_realtime, :internal ) end @@ -780,50 +760,41 @@ defmodule Plausible.Stats.Filters.QueryParserTest do test "parsing `all` with previous data", %{site: site} do site = Map.put(site, :stats_start_date, ~D[2020-01-01]) - expected_date_range = DateTimeRange.new!(~D[2020-01-01], ~D[2021-05-05], "UTC") - check_date_range(%{"date_range" => "all"}, site, %{date_range: expected_date_range}) + expected_date_range = DateTimeRange.new!(~D[2020-01-01], ~D[2021-05-05], "Etc/UTC") + check_date_range(%{"date_range" => "all"}, site, expected_date_range) end test "parsing `all` with no previous data", %{site: site} do site = Map.put(site, :stats_start_date, nil) - check_date_range(%{"date_range" => "all"}, site, %{date_range: @date_range_day}) + check_date_range(%{"date_range" => "all"}, site, @date_range_day) end test "parsing custom date range from simple date strings", %{site: site} do - check_date_range(%{"date_range" => ["2021-05-05", "2021-05-05"]}, site, %{ - date_range: @date_range_day - }) + check_date_range(%{"date_range" => ["2021-05-05", "2021-05-05"]}, site, @date_range_day) end test "parsing custom date range from iso8601 timestamps", %{site: site} do check_date_range( - %{"date_range" => ["2024-01-01T00:00:00 UTC", "2024-01-02T23:59:59 UTC"]}, + %{"date_range" => ["2024-01-01T00:00:00Z", "2024-01-02T23:59:59Z"]}, site, - %{ - date_range: - DateTimeRange.new!( - DateTime.new!(~D[2024-01-01], ~T[00:00:00], "UTC"), - DateTime.new!(~D[2024-01-02], ~T[23:59:59], "UTC") - ) - } + DateTimeRange.new!( + DateTime.new!(~D[2024-01-01], ~T[00:00:00], "Etc/UTC"), + DateTime.new!(~D[2024-01-02], ~T[23:59:59], "Etc/UTC") + ) ) check_date_range( %{ "date_range" => [ - "2024-08-29T07:12:34 America/Los_Angeles", - "2024-08-29T10:12:34 America/Los_Angeles" + "2024-08-29T07:12:34-07:00", + "2024-08-29T10:12:34-07:00" ] }, site, - %{ - date_range: - DateTimeRange.new!( - DateTime.new!(~D[2024-08-29], ~T[07:12:34], "America/Los_Angeles"), - DateTime.new!(~D[2024-08-29], ~T[10:12:34], "America/Los_Angeles") - ), - timezone: "America/Los_Angeles" - } + DateTimeRange.new!( + ~U[2024-08-29 14:12:34Z], + ~U[2024-08-29 17:12:34Z] + ) ) end @@ -845,59 +816,31 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_error( site, - "#/date_range: Invalid date range [\"2021-02-03T00:00:00\", \"2021-02-03T23:59:59\"]" - ) - end - - test "custom date range is invalid when timestamp timezones are different", %{site: site} do - %{ - "site_id" => site.domain, - "date_range" => ["2021-02-03T00:00:00 Europe/Tallinn", "2021-02-03T23:59:59 UTC"], - "metrics" => ["visitors"] - } - |> check_error( - site, - "Invalid date_range '[\"2021-02-03T00:00:00 Europe/Tallinn\", \"2021-02-03T23:59:59 UTC\"]'." + "Invalid date_range '[\"2021-02-03T00:00:00\", \"2021-02-03T23:59:59\"]'." ) end test "custom date range is invalid when timestamp timezone is invalid", %{site: site} do %{ "site_id" => site.domain, - "date_range" => ["2021-02-03T00:00:00 Fake/Timezone", "2021-02-03T23:59:59 Fake/Timezone"], + "date_range" => ["2021-02-03T00:00:00-25:00", "2021-02-03T23:59:59-25:00"], "metrics" => ["visitors"] } |> check_error( site, - "Invalid date_range '[\"2021-02-03T00:00:00 Fake/Timezone\", \"2021-02-03T23:59:59 Fake/Timezone\"]'." + "#/date_range: Invalid date range [\"2021-02-03T00:00:00-25:00\", \"2021-02-03T23:59:59-25:00\"]" ) end test "custom date range is invalid when date and timestamp are combined", %{site: site} do %{ "site_id" => site.domain, - "date_range" => ["2021-02-03T00:00:00 UTC", "2021-02-04"], + "date_range" => ["2021-02-03T00:00:00Z", "2021-02-04"], "metrics" => ["visitors"] } |> check_error( site, - "Invalid date_range '[\"2021-02-03T00:00:00 UTC\", \"2021-02-04\"]'." - ) - end - - test "custom date range is invalid when timestamp cannot be converted to datetime due to a gap in timezone", - %{site: site} do - %{ - "site_id" => site.domain, - "date_range" => [ - "2024-03-31T03:30:00 Europe/Tallinn", - "2024-04-15T10:00:00 Europe/Tallinn" - ], - "metrics" => ["visitors"] - } - |> check_error( - site, - "Invalid date_range '[\"2024-03-31T03:30:00 Europe/Tallinn\", \"2024-04-15T10:00:00 Europe/Tallinn\"]'." + "Invalid date_range '[\"2021-02-03T00:00:00Z\", \"2021-02-04\"]'." ) end @@ -914,7 +857,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do {"year", @date_range_year} ] do %{"date_range" => date_range_shortcut, "date" => date} - |> check_date_range(site, %{date_range: expected_date_range}, :internal) + |> check_date_range(site, expected_date_range, :internal) end end @@ -933,14 +876,11 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } do site = %{site | timezone: "America/Santiago"} - expected_date_range = - DateTimeRange.new!( - DateTime.new!(~D[2022-09-11], ~T[01:00:00], site.timezone), - DateTime.new!(~D[2022-09-11], ~T[23:59:59], site.timezone) - ) - %{"date_range" => ["2022-09-11", "2022-09-11"]} - |> check_date_range(site, %{date_range: expected_date_range}) + |> check_date_range( + site, + DateTimeRange.new!(~U[2022-09-11 04:00:00Z], ~U[2022-09-12 02:59:59Z]) + ) end test "parses date_range.first into the latest of ambiguous datetimes in site.timezone", %{ @@ -948,17 +888,11 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } do site = %{site | timezone: "America/Havana"} - {:ambiguous, _, expected_first_datetime} = - DateTime.new(~D[2023-11-05], ~T[00:00:00], site.timezone) - - expected_date_range = - DateTimeRange.new!( - expected_first_datetime, - DateTime.new!(~D[2023-11-05], ~T[23:59:59], site.timezone) - ) - %{"date_range" => ["2023-11-05", "2023-11-05"]} - |> check_date_range(site, %{date_range: expected_date_range}) + |> check_date_range( + site, + DateTimeRange.new!(~U[2023-11-05 05:00:00Z], ~U[2023-11-06 04:59:59Z]) + ) end test "parses date_range.last into the earliest of ambiguous datetimes in site.timezone", %{ @@ -966,17 +900,11 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } do site = %{site | timezone: "America/Asuncion"} - {:ambiguous, first_dt, _second_dt} = - DateTime.new(~D[2024-03-23], ~T[23:59:59], site.timezone) - - expected_date_range = - DateTimeRange.new!( - DateTime.new!(~D[2024-03-23], ~T[00:00:00], site.timezone), - first_dt - ) - %{"date_range" => ["2024-03-23", "2024-03-23"]} - |> check_date_range(site, %{date_range: expected_date_range}) + |> check_date_range( + site, + DateTimeRange.new!(~U[2024-03-23 03:00:00Z], ~U[2024-03-24 02:59:59Z]) + ) end end @@ -991,7 +919,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:visitors], - date_range: @date_range_day, + utc_time_range: @date_range_day, filters: [], dimensions: ["event:#{unquote(dimension)}"], order_by: nil, @@ -1012,7 +940,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:visitors], - date_range: @date_range_day, + utc_time_range: @date_range_day, filters: [], dimensions: ["visit:#{unquote(dimension)}"], order_by: nil, @@ -1032,7 +960,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:visitors], - date_range: @date_range_day, + utc_time_range: @date_range_day, filters: [], dimensions: ["event:props:foobar"], order_by: nil, @@ -1093,7 +1021,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:visitors, :events], - date_range: @date_range_day, + utc_time_range: @date_range_day, filters: [], dimensions: [], order_by: [{:events, :desc}, {:visitors, :asc}], @@ -1113,7 +1041,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:visitors], - date_range: @date_range_day, + utc_time_range: @date_range_day, filters: [], dimensions: ["event:name"], order_by: [{"event:name", :desc}], @@ -1221,7 +1149,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do # } # |> check_success(site, %{ # metrics: [:conversion_rate], - # date_range: @date_range_day, + # utc_time_range: @date_range_day, # filters: [[:is, "event:goal", [event: "Signup"]]], # dimensions: [], # order_by: nil, @@ -1241,7 +1169,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do # } # |> check_success(site, %{ # metrics: [:conversion_rate], - # date_range: @date_range_day, + # utc_time_range: @date_range_day, # filters: [], # dimensions: ["event:goal"], # order_by: nil, @@ -1261,7 +1189,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:conversion_rate, :group_conversion_rate], - date_range: @date_range_day, + utc_time_range: @date_range_day, filters: [ [:is, "event:props:foo", ["bar"]] ], @@ -1299,7 +1227,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do # } # |> check_success(site, %{ # metrics: [:views_per_visit], - # date_range: @date_range_day, + # utc_time_range: @date_range_day, # filters: [[:is, "event:goal", [event: "Signup"]]], # dimensions: [], # order_by: nil, @@ -1346,7 +1274,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:bounce_rate], - date_range: @date_range_day, + utc_time_range: @date_range_day, filters: [], dimensions: ["visit:device"], order_by: nil, @@ -1378,7 +1306,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:bounce_rate], - date_range: @date_range_day, + utc_time_range: @date_range_day, filters: [], dimensions: ["event:page"], order_by: nil, @@ -1397,7 +1325,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:bounce_rate], - date_range: @date_range_day, + utc_time_range: @date_range_day, filters: [[:is, "event:props:foo", ["(none)"]]], dimensions: [], order_by: nil, diff --git a/test/plausible/stats/query_result_test.exs b/test/plausible/stats/query_result_test.exs index b06f296fdb09..3782a0e9c796 100644 --- a/test/plausible/stats/query_result_test.exs +++ b/test/plausible/stats/query_result_test.exs @@ -51,8 +51,8 @@ defmodule Plausible.Stats.QueryResultTest do "pageviews" ], "date_range": [ - "2024-01-01T00:00:00 UTC", - "2024-02-01T23:59:59 UTC" + "2024-01-01T00:00:00+00:00", + "2024-02-01T23:59:59+00:00" ], "filters": [], "dimensions": [], diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exs index 9634e1aece5d..89425a62e8b5 100644 --- a/test/plausible/stats/query_test.exs +++ b/test/plausible/stats/query_test.exs @@ -1,6 +1,6 @@ defmodule Plausible.Stats.QueryTest do use Plausible.DataCase, async: true - alias Plausible.Stats.{Query, DateTimeRange} + alias Plausible.Stats.Query alias Plausible.Stats.Legacy.QueryBuilder doctest Plausible.Stats.Legacy.QueryBuilder @@ -12,12 +12,15 @@ defmodule Plausible.Stats.QueryTest do insert(:site, members: [user], inserted_at: ~N[2020-01-01T00:00:00], - stats_start_date: ~D[2020-01-01] + stats_start_date: ~D[2020-01-01], + timezone: "US/Eastern" ) {:ok, site: site, user: user} end + @now ~U[2024-05-03 16:30:00Z] + @tag :slow test "keeps current timestamp so that utc_boundaries don't depend on time passing by", %{ site: site @@ -32,193 +35,130 @@ defmodule Plausible.Stats.QueryTest do end test "parses day format", %{site: site} do - q = Query.from(site, %{"period" => "day", "date" => "2019-01-01"}) + q = Query.from(site, %{"period" => "day", "date" => "2019-01-01"}, %{}, @now) - assert q.date_range.first == DateTime.new!(~D[2019-01-01], ~T[00:00:00], site.timezone) - assert q.date_range.last == DateTime.new!(~D[2019-01-01], ~T[23:59:59], site.timezone) + assert q.utc_time_range.first == ~U[2019-01-01 05:00:00Z] + assert q.utc_time_range.last == ~U[2019-01-02 04:59:59Z] assert q.interval == "hour" end test "day format defaults to today", %{site: site} do - q = Query.from(site, %{"period" => "day"}) - - expected_first_datetime = Date.utc_today() |> DateTime.new!(~T[00:00:00], site.timezone) - expected_last_datetime = Date.utc_today() |> DateTime.new!(~T[23:59:59], site.timezone) + q = Query.from(site, %{"period" => "day"}, %{}, @now) - assert q.date_range.first == expected_first_datetime - assert q.date_range.last == expected_last_datetime + assert q.utc_time_range.first == ~U[2024-05-03 04:00:00Z] + assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z] assert q.interval == "hour" end test "parses realtime format", %{site: site} do - q = Query.from(site, %{"period" => "realtime"}) - - utc_now = DateTime.shift_zone!(q.now, "Etc/UTC") + q = Query.from(site, %{"period" => "realtime"}, %{}, @now) - expected_first_datetime = utc_now |> DateTime.shift(minute: -5) - expected_last_datetime = utc_now |> DateTime.shift(second: 5) - - assert q.date_range.first == expected_first_datetime - assert q.date_range.last == expected_last_datetime + assert q.utc_time_range.first == ~U[2024-05-03 16:25:00Z] + assert q.utc_time_range.last == ~U[2024-05-03 16:30:05Z] assert q.period == "realtime" end test "parses month format", %{site: site} do - q = Query.from(site, %{"period" => "month", "date" => "2019-01-01"}) + q = Query.from(site, %{"period" => "month", "date" => "2019-01-01"}, %{}, @now) - assert q.date_range.first == DateTime.new!(~D[2019-01-01], ~T[00:00:00], site.timezone) - assert q.date_range.last == DateTime.new!(~D[2019-01-31], ~T[23:59:59], site.timezone) + assert q.utc_time_range.first == ~U[2019-01-01 05:00:00Z] + assert q.utc_time_range.last == ~U[2019-02-01 04:59:59Z] assert q.interval == "day" end test "parses 6 month format", %{site: site} do - q = Query.from(site, %{"period" => "6mo"}) - - expected_first_datetime = - q.now - |> DateTime.to_date() - |> Date.shift(month: -5) - |> Date.beginning_of_month() - |> DateTime.new!(~T[00:00:00], site.timezone) - - expected_last_datetime = - q.now - |> DateTime.to_date() - |> Date.end_of_month() - |> DateTime.new!(~T[23:59:59], site.timezone) - - assert q.date_range.first == expected_first_datetime - assert q.date_range.last == expected_last_datetime + q = Query.from(site, %{"period" => "6mo"}, %{}, @now) + + assert q.utc_time_range.first == ~U[2023-12-01 05:00:00Z] + assert q.utc_time_range.last == ~U[2024-06-01 03:59:59Z] assert q.interval == "month" end test "parses 12 month format", %{site: site} do - q = Query.from(site, %{"period" => "12mo"}) - - expected_first_datetime = - q.now - |> DateTime.to_date() - |> Date.shift(month: -11) - |> Date.beginning_of_month() - |> DateTime.new!(~T[00:00:00], site.timezone) - - expected_last_datetime = - q.now - |> DateTime.to_date() - |> Date.end_of_month() - |> DateTime.new!(~T[23:59:59], site.timezone) - - assert q.date_range.first == expected_first_datetime - assert q.date_range.last == expected_last_datetime + q = Query.from(site, %{"period" => "12mo"}, %{}, @now) + + assert q.utc_time_range.first == ~U[2023-06-01 04:00:00Z] + assert q.utc_time_range.last == ~U[2024-06-01 03:59:59Z] assert q.interval == "month" end test "parses year to date format", %{site: site} do - q = Query.from(site, %{"period" => "year"}) - - %Date{year: current_year} = DateTime.to_date(q.now) - - expected_first_datetime = - Date.new!(current_year, 1, 1) - |> DateTime.new!(~T[00:00:00], site.timezone) + q = Query.from(site, %{"period" => "year"}, %{}, @now) - expected_last_datetime = - Date.new!(current_year, 12, 31) - |> DateTime.new!(~T[23:59:59], site.timezone) - - assert q.date_range.first == expected_first_datetime - assert q.date_range.last == expected_last_datetime + assert q.utc_time_range.first == ~U[2024-01-01 05:00:00Z] + assert q.utc_time_range.last == ~U[2025-01-01 04:59:59Z] assert q.interval == "month" end test "parses all time", %{site: site} do - q = Query.from(site, %{"period" => "all"}) - - expected_last_datetime = - q.now - |> DateTime.to_date() - |> DateTime.new!(~T[23:59:59], site.timezone) + q = Query.from(site, %{"period" => "all"}, %{}, @now) - assert DateTime.to_naive(q.date_range.first) == site.inserted_at - assert q.date_range.last == expected_last_datetime + assert q.utc_time_range.first == ~U[2020-01-01 05:00:00Z] + assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z] assert q.period == "all" assert q.interval == "month" end - test "parses all time in site timezone", %{site: site} do - for timezone <- ["Etc/GMT+12", "Etc/GMT-12"] do - site = Map.put(site, :timezone, timezone) - query = Query.from(site, %{"period" => "all"}) + test "parses all time in GMT+12 timezone", %{site: site} do + site = Map.put(site, :timezone, "Etc/GMT+12") + q = Query.from(site, %{"period" => "all"}, %{}, @now) - expected_first_datetime = DateTime.new!(~D[2020-01-01], ~T[00:00:00], site.timezone) - - expected_last_datetime = - DateTime.now!(site.timezone) - |> DateTime.to_date() - |> DateTime.new!(~T[23:59:59], site.timezone) - - assert query.date_range.first == expected_first_datetime - assert query.date_range.last == expected_last_datetime - end + assert q.utc_time_range.first == ~U[2020-01-01 12:00:00Z] + assert q.utc_time_range.last == ~U[2024-05-04 11:59:59Z] end test "all time shows today if site has no start date", %{site: site} do site = Map.put(site, :stats_start_date, nil) - q = Query.from(site, %{"period" => "all"}) + q = Query.from(site, %{"period" => "all"}, %{}, @now) - today = Date.utc_today() - - assert q.date_range == DateTimeRange.new!(today, today, site.timezone) + assert q.utc_time_range.first == ~U[2024-05-03 04:00:00Z] + assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z] assert q.period == "all" assert q.interval == "hour" end test "all time shows hourly if site is completely new", %{site: site} do - site = Map.put(site, :stats_start_date, Date.utc_today()) - q = Query.from(site, %{"period" => "all"}) - - today = Date.utc_today() + site = Map.put(site, :stats_start_date, @now |> DateTime.to_date()) + q = Query.from(site, %{"period" => "all"}, %{}, @now) - assert q.date_range == DateTimeRange.new!(today, today, site.timezone) + assert q.utc_time_range.first == ~U[2024-05-03 04:00:00Z] + assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z] assert q.period == "all" assert q.interval == "hour" end test "all time shows daily if site is more than a day old", %{site: site} do - today = Date.utc_today() - yesterday = today |> Date.shift(day: -1) - + yesterday = @now |> DateTime.to_date() |> Date.shift(day: -1) site = Map.put(site, :stats_start_date, yesterday) - q = Query.from(site, %{"period" => "all"}) + q = Query.from(site, %{"period" => "all"}, %{}, @now) - assert q.date_range == DateTimeRange.new!(yesterday, today, site.timezone) + assert q.utc_time_range.first == ~U[2024-05-02 04:00:00Z] + assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z] assert q.period == "all" assert q.interval == "day" end test "all time shows monthly if site is more than a month old", %{site: site} do - today = Date.utc_today() - last_month = today |> Date.shift(month: -1) - + last_month = @now |> DateTime.to_date() |> Date.shift(month: -1) site = Map.put(site, :stats_start_date, last_month) - q = Query.from(site, %{"period" => "all"}) + q = Query.from(site, %{"period" => "all"}, %{}, @now) - assert q.date_range == DateTimeRange.new!(last_month, today, site.timezone) + assert q.utc_time_range.first == ~U[2024-04-03 04:00:00Z] + assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z] assert q.period == "all" assert q.interval == "month" end test "all time uses passed interval different from the default interval", %{site: site} do - today = Date.utc_today() - last_month = today |> Date.shift(month: -1) - + last_month = @now |> DateTime.to_date() |> Date.shift(month: -1) site = Map.put(site, :stats_start_date, last_month) - q = Query.from(site, %{"period" => "all", "interval" => "week"}) + q = Query.from(site, %{"period" => "all", "interval" => "week"}, %{}, @now) - assert q.date_range == DateTimeRange.new!(last_month, today, site.timezone) + assert q.utc_time_range.first == ~U[2024-04-03 04:00:00Z] + assert q.utc_time_range.last == ~U[2024-05-04 03:59:59Z] assert q.period == "all" assert q.interval == "week" end @@ -228,10 +168,16 @@ defmodule Plausible.Stats.QueryTest do end test "parses custom format", %{site: site} do - q = Query.from(site, %{"period" => "custom", "from" => "2019-01-01", "to" => "2019-01-15"}) + q = + Query.from( + site, + %{"period" => "custom", "from" => "2019-01-01", "to" => "2019-01-15"}, + %{}, + @now + ) - assert q.date_range.first == DateTime.new!(~D[2019-01-01], ~T[00:00:00], site.timezone) - assert q.date_range.last == DateTime.new!(~D[2019-01-15], ~T[23:59:59], site.timezone) + assert q.utc_time_range.first == ~U[2019-01-01 05:00:00Z] + assert q.utc_time_range.last == ~U[2019-01-16 04:59:59Z] assert q.interval == "day" end diff --git a/test/plausible/stats/time_test.exs b/test/plausible/stats/time_test.exs index 37cb983fb938..ba3dfb7a462b 100644 --- a/test/plausible/stats/time_test.exs +++ b/test/plausible/stats/time_test.exs @@ -8,7 +8,8 @@ defmodule Plausible.Stats.TimeTest do test "with time:month dimension" do assert time_labels(%{ dimensions: ["visit:device", "time:month"], - date_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-02-01], "UTC") + utc_time_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-02-01], "UTC"), + timezone: "UTC" }) == [ "2022-01-01", "2022-02-01" @@ -16,7 +17,8 @@ defmodule Plausible.Stats.TimeTest do assert time_labels(%{ dimensions: ["visit:device", "time:month"], - date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-03-07], "UTC") + utc_time_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-03-07], "UTC"), + timezone: "UTC" }) == [ "2022-01-01", "2022-02-01", @@ -27,7 +29,8 @@ defmodule Plausible.Stats.TimeTest do test "with time:week dimension" do assert time_labels(%{ dimensions: ["time:week"], - date_range: DateTimeRange.new!(~D[2020-12-20], ~D[2021-01-08], "UTC") + utc_time_range: DateTimeRange.new!(~D[2020-12-20], ~D[2021-01-08], "UTC"), + timezone: "UTC" }) == [ "2020-12-20", "2020-12-21", @@ -37,7 +40,8 @@ defmodule Plausible.Stats.TimeTest do assert time_labels(%{ dimensions: ["time:week"], - date_range: DateTimeRange.new!(~D[2020-12-21], ~D[2021-01-03], "UTC") + utc_time_range: DateTimeRange.new!(~D[2020-12-21], ~D[2021-01-03], "UTC"), + timezone: "UTC" }) == [ "2020-12-21", "2020-12-28" @@ -47,7 +51,8 @@ defmodule Plausible.Stats.TimeTest do test "with time:day dimension" do assert time_labels(%{ dimensions: ["time:day"], - date_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-02-02], "UTC") + utc_time_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-02-02], "UTC"), + timezone: "UTC" }) == [ "2022-01-17", "2022-01-18", @@ -72,7 +77,8 @@ defmodule Plausible.Stats.TimeTest do test "with time:hour dimension" do assert time_labels(%{ dimensions: ["time:hour"], - date_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-01-17], "UTC") + utc_time_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-01-17], "UTC"), + timezone: "UTC" }) == [ "2022-01-17 00:00:00", "2022-01-17 01:00:00", @@ -102,7 +108,8 @@ defmodule Plausible.Stats.TimeTest do assert time_labels(%{ dimensions: ["time:hour"], - date_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-01-18], "UTC") + utc_time_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-01-18], "UTC"), + timezone: "UTC" }) == [ "2022-01-17 00:00:00", "2022-01-17 01:00:00", @@ -154,51 +161,75 @@ defmodule Plausible.Stats.TimeTest do "2022-01-18 23:00:00" ] end - end - test "with time:minute dimension" do - now = DateTime.new!(~D[2024-01-01], ~T[12:30:57], "UTC") + test "with a different time range" do + {:ok, from_timestamp, _} = DateTime.from_iso8601("2024-09-04T21:53:17+03:00") + {:ok, to_timestamp, _} = DateTime.from_iso8601("2024-09-05T05:59:59+03:00") + + assert time_labels(%{ + dimensions: ["time:hour"], + utc_time_range: + DateTimeRange.new!(from_timestamp, to_timestamp) + |> DateTimeRange.to_timezone("Etc/UTC"), + timezone: "Europe/Tallinn" + }) == [ + "2024-09-04 21:00:00", + "2024-09-04 22:00:00", + "2024-09-04 23:00:00", + "2024-09-05 00:00:00", + "2024-09-05 01:00:00", + "2024-09-05 02:00:00", + "2024-09-05 03:00:00", + "2024-09-05 04:00:00", + "2024-09-05 05:00:00" + ] + end + + test "with time:minute dimension" do + now = DateTime.new!(~D[2024-01-01], ~T[12:30:57], "UTC") - # ~U[2024-01-01 12:00:57Z] - first_dt = DateTime.shift(now, minute: -30) - # ~U[2024-01-01 12:31:02Z] - last_dt = DateTime.shift(now, second: 5) + # ~U[2024-01-01 12:00:57Z] + first_dt = DateTime.shift(now, minute: -30) + # ~U[2024-01-01 12:31:02Z] + last_dt = DateTime.shift(now, second: 5) - assert time_labels(%{ - dimensions: ["time:minute"], - now: now, - date_range: DateTimeRange.new!(first_dt, last_dt) - }) == [ - "2024-01-01 12:00:00", - "2024-01-01 12:01:00", - "2024-01-01 12:02:00", - "2024-01-01 12:03:00", - "2024-01-01 12:04:00", - "2024-01-01 12:05:00", - "2024-01-01 12:06:00", - "2024-01-01 12:07:00", - "2024-01-01 12:08:00", - "2024-01-01 12:09:00", - "2024-01-01 12:10:00", - "2024-01-01 12:11:00", - "2024-01-01 12:12:00", - "2024-01-01 12:13:00", - "2024-01-01 12:14:00", - "2024-01-01 12:15:00", - "2024-01-01 12:16:00", - "2024-01-01 12:17:00", - "2024-01-01 12:18:00", - "2024-01-01 12:19:00", - "2024-01-01 12:20:00", - "2024-01-01 12:21:00", - "2024-01-01 12:22:00", - "2024-01-01 12:23:00", - "2024-01-01 12:24:00", - "2024-01-01 12:25:00", - "2024-01-01 12:26:00", - "2024-01-01 12:27:00", - "2024-01-01 12:28:00", - "2024-01-01 12:29:00" - ] + assert time_labels(%{ + dimensions: ["time:minute"], + now: now, + utc_time_range: DateTimeRange.new!(first_dt, last_dt), + timezone: "UTC" + }) == [ + "2024-01-01 12:00:00", + "2024-01-01 12:01:00", + "2024-01-01 12:02:00", + "2024-01-01 12:03:00", + "2024-01-01 12:04:00", + "2024-01-01 12:05:00", + "2024-01-01 12:06:00", + "2024-01-01 12:07:00", + "2024-01-01 12:08:00", + "2024-01-01 12:09:00", + "2024-01-01 12:10:00", + "2024-01-01 12:11:00", + "2024-01-01 12:12:00", + "2024-01-01 12:13:00", + "2024-01-01 12:14:00", + "2024-01-01 12:15:00", + "2024-01-01 12:16:00", + "2024-01-01 12:17:00", + "2024-01-01 12:18:00", + "2024-01-01 12:19:00", + "2024-01-01 12:20:00", + "2024-01-01 12:21:00", + "2024-01-01 12:22:00", + "2024-01-01 12:23:00", + "2024-01-01 12:24:00", + "2024-01-01 12:25:00", + "2024-01-01 12:26:00", + "2024-01-01 12:27:00", + "2024-01-01 12:28:00", + "2024-01-01 12:29:00" + ] + end end end