APIv2: Standard iso8601 timestamps, operate on UTC (plausible#4563)

* query.date_range is now in UTC instead of user timezone

This simplifies things down the line and fixes several bugs where
query.date_range is cast to naivedatetime for ecto purposes

Many places still remain broken:
- comparison queries
- `to_date_range` calls

* Make default_for_date_range not care about time zones

* Make timezone parameter mandatory for to_date_range

* Simplify utc_date_range, update legacy query builder

* Fix more cases where query date range is needed

* query.date_range -> query.utc_time_range

* Query.date_range/1 function

* ensure_include_imported update

* Clean up send_email_report

lib/plausible/google/api.ex

@@ -7,7 +7,7 @@ defmodule Plausible.Google.API do
 
   alias Plausible.Google.HTTP
   alias Plausible.Google.SearchConsole
-  alias Plausible.Stats.DateTimeRange
+  alias Plausible.Stats.Query
 
   require Logger
 
@@ -65,12 +65,11 @@ defmodule Plausible.Google.API do
          {:ok, access_token} <- maybe_refresh_token(site.google_auth),
          {:ok, gsc_filters} <-
            SearchConsole.Filters.transform(site.google_auth.property, query.filters, search),
-         date_range = DateTimeRange.to_date_range(query.date_range),
          {:ok, stats} <-
            HTTP.list_stats(
              access_token,
              site.google_auth.property,
-             date_range,
+             Query.date_range(query),
              pagination,
              gsc_filters
            ) do

lib/plausible/stats/breakdown.ex

@@ -116,16 +116,16 @@ defmodule Plausible.Stats.Breakdown do
       from e in subquery(timed_page_transitions_q),
         group_by: e.pathname
 
+    date_range = Query.date_range(query)
+
     timed_pages_q =
       if query.include_imported do
         # Imported page views have pre-calculated values
         imported_timed_pages_q =
           from i in "imported_pages",
             group_by: i.page,
             where: i.site_id == ^site.id,
-            where:
-              i.date >= ^DateTime.to_naive(query.date_range.first) and
-                i.date <= ^DateTime.to_naive(query.date_range.last),
+            where: i.date >= ^date_range.first and i.date <= ^date_range.last,
             where: i.page in ^pages,
             select: %{
               page: i.page,

lib/plausible/stats/comparisons.ex

@@ -64,15 +64,17 @@ defmodule Plausible.Stats.Comparisons do
       |> Keyword.put_new(:now, DateTime.now!(site.timezone))
       |> Keyword.put_new(:match_day_of_week?, false)
 
-    source_date_range = DateTimeRange.to_date_range(source_query.date_range)
+    source_date_range = Query.date_range(source_query)
 
     with :ok <- validate_mode(source_query, mode),
          {:ok, comparison_date_range} <- get_comparison_date_range(source_date_range, mode, opts) do
-      %Date.Range{first: first, last: last} = comparison_date_range
+      new_range =
+        DateTimeRange.new!(comparison_date_range.first, comparison_date_range.last, site.timezone)
+        |> DateTimeRange.to_timezone("Etc/UTC")
 
       comparison_query =
         source_query
-        |> Query.set(date_range: DateTimeRange.new!(first, last, site.timezone))
+        |> Query.set(utc_time_range: new_range)
         |> maybe_include_imported(source_query)
 
       {:ok, comparison_query}

lib/plausible/stats/datetime_range.ex

@@ -45,7 +45,16 @@ defmodule Plausible.Stats.DateTimeRange do
     %__MODULE__{first: first, last: last}
   end
 
-  def to_date_range(%__MODULE__{first: first, last: last}) do
+  def to_timezone(%__MODULE__{first: first, last: last}, timezone) do
+    first = DateTime.shift_zone!(first, timezone)
+    last = DateTime.shift_zone!(last, timezone)
+
+    %__MODULE__{first: first, last: last}
+  end
+
+  def to_date_range(datetime_range, timezone) do
+    %__MODULE__{first: first, last: last} = to_timezone(datetime_range, timezone)
+
     first = DateTime.to_date(first)
     last = DateTime.to_date(last)
 

lib/plausible/stats/filters/query_parser.ex

@@ -18,7 +18,9 @@ defmodule Plausible.Stats.Filters.QueryParser do
 
     with :ok <- JSONSchema.validate(schema_type, params),
          {:ok, date} <- parse_date(site, Map.get(params, "date"), date),
-         {:ok, date_range} <- parse_date_range(site, Map.get(params, "date_range"), date, now),
+         {:ok, raw_time_range} <-
+           parse_time_range(site, Map.get(params, "date_range"), date, now),
+         utc_time_range = raw_time_range |> DateTimeRange.to_timezone("Etc/UTC"),
          {:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])),
          {:ok, filters} <- parse_filters(Map.get(params, "filters", [])),
          {:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])),
@@ -28,10 +30,10 @@ defmodule Plausible.Stats.Filters.QueryParser do
          query = %{
            metrics: metrics,
            filters: filters,
-           date_range: date_range,
+           utc_time_range: utc_time_range,
            dimensions: dimensions,
            order_by: order_by,
-           timezone: date_range.first.time_zone,
+           timezone: site.timezone,
            preloaded_goals: preloaded_goals,
            include: include
          },
@@ -151,7 +153,7 @@ defmodule Plausible.Stats.Filters.QueryParser do
     {:ok, date}
   end
 
-  defp parse_date_range(_site, date_range, _date, now) when date_range in ["realtime", "30m"] do
+  defp parse_time_range(_site, date_range, _date, now) when date_range in ["realtime", "30m"] do
     duration_minutes =
       case date_range do
         "realtime" -> 5
@@ -164,27 +166,27 @@ defmodule Plausible.Stats.Filters.QueryParser do
     {:ok, DateTimeRange.new!(first_datetime, last_datetime)}
   end
 
-  defp parse_date_range(site, "day", date, _now) do
+  defp parse_time_range(site, "day", date, _now) do
     {:ok, DateTimeRange.new!(date, date, site.timezone)}
   end
 
-  defp parse_date_range(site, "7d", date, _now) do
+  defp parse_time_range(site, "7d", date, _now) do
     first = date |> Date.add(-6)
     {:ok, DateTimeRange.new!(first, date, site.timezone)}
   end
 
-  defp parse_date_range(site, "30d", date, _now) do
+  defp parse_time_range(site, "30d", date, _now) do
     first = date |> Date.add(-30)
     {:ok, DateTimeRange.new!(first, date, site.timezone)}
   end
 
-  defp parse_date_range(site, "month", date, _now) do
+  defp parse_time_range(site, "month", date, _now) do
     last = date |> Date.end_of_month()
     first = last |> Date.beginning_of_month()
     {:ok, DateTimeRange.new!(first, last, site.timezone)}
   end
 
-  defp parse_date_range(site, "6mo", date, _now) do
+  defp parse_time_range(site, "6mo", date, _now) do
     last = date |> Date.end_of_month()
 
     first =
@@ -195,7 +197,7 @@ defmodule Plausible.Stats.Filters.QueryParser do
     {:ok, DateTimeRange.new!(first, last, site.timezone)}
   end
 
-  defp parse_date_range(site, "12mo", date, _now) do
+  defp parse_time_range(site, "12mo", date, _now) do
     last = date |> Date.end_of_month()
 
     first =
@@ -206,27 +208,27 @@ defmodule Plausible.Stats.Filters.QueryParser do
     {:ok, DateTimeRange.new!(first, last, site.timezone)}
   end
 
-  defp parse_date_range(site, "year", date, _now) do
+  defp parse_time_range(site, "year", date, _now) do
     last = date |> Timex.end_of_year()
     first = last |> Timex.beginning_of_year()
     {:ok, DateTimeRange.new!(first, last, site.timezone)}
   end
 
-  defp parse_date_range(site, "all", date, _now) do
+  defp parse_time_range(site, "all", date, _now) do
     start_date = Plausible.Sites.stats_start_date(site) || date
 
     {:ok, DateTimeRange.new!(start_date, date, site.timezone)}
   end
 
-  defp parse_date_range(site, [from, to], _date, _now)
+  defp parse_time_range(site, [from, to], _date, _now)
        when is_binary(from) and is_binary(to) do
     case date_range_from_date_strings(site, from, to) do
       {:ok, date_range} -> {:ok, date_range}
       {:error, _} -> date_range_from_timestamps(from, to)
     end
   end
 
-  defp parse_date_range(_site, unknown, _date, _now),
+  defp parse_time_range(_site, unknown, _date, _now),
     do: {:error, "Invalid date_range '#{i(unknown)}'."}
 
   defp date_range_from_date_strings(site, from, to) do
@@ -237,22 +239,14 @@ defmodule Plausible.Stats.Filters.QueryParser do
   end
 
   defp date_range_from_timestamps(from, to) do
-    with {:ok, from_datetime} <- datetime_from_timestamp(from),
-         {:ok, to_datetime} <- datetime_from_timestamp(to),
-         true <- from_datetime.time_zone == to_datetime.time_zone do
+    with {:ok, from_datetime, _offset} <- DateTime.from_iso8601(from),
+         {:ok, to_datetime, _offset} <- DateTime.from_iso8601(to) do
       {:ok, DateTimeRange.new!(from_datetime, to_datetime)}
     else
       _ -> {:error, "Invalid date_range '#{i([from, to])}'."}
     end
   end
 
-  defp datetime_from_timestamp(timestamp_string) do
-    with [timestamp, timezone] <- String.split(timestamp_string),
-         {:ok, naive_datetime} <- NaiveDateTime.from_iso8601(timestamp) do
-      DateTime.from_naive(naive_datetime, timezone)
-    end
-  end
-
   defp today(site), do: DateTime.now!(site.timezone) |> DateTime.to_date()
 
   defp parse_dimensions(dimensions) when is_list(dimensions) do

lib/plausible/stats/goal_suggestions.ex

@@ -57,13 +57,13 @@ defmodule Plausible.Stats.GoalSuggestions do
       )
       |> maybe_set_limit(limit)
 
+    date_range = Query.date_range(query)
+
     imported_q =
       from(i in "imported_custom_events",
         where: i.site_id == ^site.id,
         where: i.import_id in ^site.complete_import_ids,
-        where:
-          i.date >= ^DateTime.to_naive(query.date_range.first) and
-            i.date <= ^DateTime.to_naive(query.date_range.last),
+        where: i.date >= ^date_range.first and i.date <= ^date_range.last,
         where: i.visitors > 0,
         where: fragment("? ilike ?", i.name, ^matches),
         where: fragment("trim(?)", i.name) != "",

lib/plausible/stats/imported/base.ex

@@ -6,7 +6,7 @@ defmodule Plausible.Stats.Imported.Base do
   import Ecto.Query
 
   alias Plausible.Imported
-  alias Plausible.Stats.{Query, DateTimeRange}
+  alias Plausible.Stats.Query
 
   import Plausible.Stats.Filters, only: [dimensions_used_in_filters: 1]
 
@@ -59,7 +59,8 @@ defmodule Plausible.Stats.Imported.Base do
 
   def query_imported(table, site, query) do
     import_ids = site.complete_import_ids
-    %{first: date_from, last: date_to} = DateTimeRange.to_date_range(query.date_range)
+    # Assumption: dates in imported table are in user-local timezone.
+    %{first: date_from, last: date_to} = Query.date_range(query)
 
     from(i in table,
       where: i.site_id == ^site.id,

lib/plausible/stats/imported/sql/expression.ex

@@ -9,6 +9,8 @@ defmodule Plausible.Stats.Imported.SQL.Expression do
   import Plausible.Stats.Util, only: [shortname: 2]
   import Ecto.Query
 
+  alias Plausible.Stats.Query
+
   @no_ref "Direct / None"
   @not_set "(not set)"
   @none "(none)"
@@ -292,8 +294,10 @@ defmodule Plausible.Stats.Imported.SQL.Expression do
   end
 
   defp select_group_fields(q, "time:week", key, query) do
+    date_range = Query.date_range(query)
+
     select_merge_as(q, [i], %{
-      key => weekstart_not_before(i.date, ^DateTime.to_naive(query.date_range.first))
+      key => weekstart_not_before(i.date, ^date_range.first)
     })
   end
 

lib/plausible/stats/interval.ex

@@ -40,9 +40,7 @@ defmodule Plausible.Stats.Interval do
   @doc """
   Returns the suggested interval for the given `DateTimeRange` struct.
   """
-  def default_for_date_range(%DateTimeRange{} = date_range) do
-    %Date.Range{first: first, last: last} = DateTimeRange.to_date_range(date_range)
-
+  def default_for_date_range(%DateTimeRange{first: first, last: last}) do
     cond do
       Timex.diff(last, first, :months) > 0 ->
         "month"