diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 7f186e477..22e04da22 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -7,6 +7,7 @@ defmodule Plausible.Google.API do alias Plausible.Google.HTTP alias Plausible.Google.SearchConsole + alias Plausible.Stats.DateTimeRange require Logger @@ -64,11 +65,12 @@ defmodule Plausible.Google.API do {:ok, access_token} <- maybe_refresh_token(site.google_auth), {:ok, gsc_filters} <- SearchConsole.Filters.transform(site.google_auth.property, query.filters, search), + date_range = DateTimeRange.to_date_range(query.date_range), {:ok, stats} <- HTTP.list_stats( access_token, site.google_auth.property, - query.date_range, + date_range, pagination, gsc_filters ) do diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 37481f0e4..15b8964e4 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -117,7 +117,9 @@ defmodule Plausible.Stats.Breakdown do from i in "imported_pages", group_by: i.page, where: i.site_id == ^site.id, - where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last, + where: + i.date >= ^DateTime.to_naive(query.date_range.first) and + i.date <= ^DateTime.to_naive(query.date_range.last), where: i.page in ^pages, select: %{ page: i.page, diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex index 69437de69..219051e68 100644 --- a/lib/plausible/stats/clickhouse.ex +++ b/lib/plausible/stats/clickhouse.ex @@ -6,7 +6,6 @@ defmodule Plausible.Stats.Clickhouse do import Ecto.Query, only: [from: 2, dynamic: 1, dynamic: 2] - alias Plausible.Stats.Query alias Plausible.Timezones @spec pageview_start_date_local(Plausible.Site.t()) :: Date.t() | nil @@ -78,7 +77,7 @@ defmodule Plausible.Stats.Clickhouse do def top_sources_for_spike(site, query, limit, page) do offset = (page - 1) * limit - {first_datetime, last_datetime} = utc_boundaries(query, site) + {first_datetime, last_datetime} = Plausible.Stats.Time.utc_boundaries(query, site) referrers = from(s in "sessions_v2", @@ -251,52 +250,4 @@ defmodule Plausible.Stats.Clickhouse do } end end - - defp utc_boundaries(%Query{now: now, period: "30m"}, site) do - last_datetime = now |> NaiveDateTime.truncate(:second) - - first_datetime = - last_datetime - |> NaiveDateTime.shift(minute: -30) - |> beginning_of_time(site.native_stats_start_at) - |> NaiveDateTime.truncate(:second) - - {first_datetime, last_datetime} - end - - defp utc_boundaries(%Query{now: now, period: "realtime"}, site) do - last_datetime = now |> NaiveDateTime.truncate(:second) - - first_datetime = - last_datetime - |> NaiveDateTime.shift(minute: -5) - |> beginning_of_time(site.native_stats_start_at) - |> NaiveDateTime.truncate(:second) - - {first_datetime, last_datetime} - end - - defp utc_boundaries(%Query{date_range: date_range}, site) do - {:ok, first} = NaiveDateTime.new(date_range.first, ~T[00:00:00]) - - first_datetime = - first - |> Timezones.to_utc_datetime(site.timezone) - |> beginning_of_time(site.native_stats_start_at) - - {:ok, last} = NaiveDateTime.new(date_range.last |> Date.shift(day: 1), ~T[00:00:00]) - - last_datetime = - Timezones.to_utc_datetime(last, site.timezone) - - {first_datetime, last_datetime} - end - - defp beginning_of_time(candidate, site_creation_date) do - if Timex.after?(site_creation_date, candidate) do - site_creation_date - else - candidate - end - end end diff --git a/lib/plausible/stats/comparisons.ex b/lib/plausible/stats/comparisons.ex index 157be97c1..23f257286 100644 --- a/lib/plausible/stats/comparisons.ex +++ b/lib/plausible/stats/comparisons.ex @@ -8,7 +8,7 @@ defmodule Plausible.Stats.Comparisons do """ alias Plausible.Stats - alias Plausible.Stats.Query + alias Plausible.Stats.{Query, DateTimeRange} @modes ~w(previous_period year_over_year custom) @disallowed_periods ~w(realtime all) @@ -21,6 +21,9 @@ defmodule Plausible.Stats.Comparisons do @doc """ Generates a comparison query based on the source query and comparison mode. + Currently only historical periods are supported for comparisons (not `realtime` + and `30m` periods). + The mode parameter specifies the type of comparison and can be one of the following: @@ -61,52 +64,55 @@ defmodule Plausible.Stats.Comparisons do |> Keyword.put_new(:now, Timex.now(site.timezone)) |> Keyword.put_new(:match_day_of_week?, false) + source_date_range = DateTimeRange.to_date_range(source_query.date_range) + with :ok <- validate_mode(source_query, mode), - {:ok, comparison_query} <- do_compare(source_query, mode, opts), - comparison_query <- maybe_include_imported(comparison_query, source_query), - do: {:ok, comparison_query} + {:ok, comparison_date_range} <- get_comparison_date_range(source_date_range, mode, opts) do + %Date.Range{first: first, last: last} = comparison_date_range + + comparison_query = + source_query + |> Query.set(date_range: DateTimeRange.new!(first, last, site.timezone)) + |> maybe_include_imported(source_query) + + {:ok, comparison_query} + end end - defp do_compare(source_query, "year_over_year", opts) do + defp get_comparison_date_range(source_date_range, "year_over_year", opts) do now = Keyword.fetch!(opts, :now) - start_date = Date.add(source_query.date_range.first, -365) - end_date = earliest(source_query.date_range.last, now) |> Date.add(-365) + start_date = Date.add(source_date_range.first, -365) + end_date = earliest(source_date_range.last, now) |> Date.add(-365) - range = Date.range(start_date, end_date) + comparison_date_range = + Date.range(start_date, end_date) + |> maybe_match_day_of_week(source_date_range, opts) - comparison_query = - source_query - |> Map.put(:date_range, range) - |> maybe_match_day_of_week(source_query, opts) - - {:ok, comparison_query} + {:ok, comparison_date_range} end - defp do_compare(source_query, "previous_period", opts) do + defp get_comparison_date_range(source_date_range, "previous_period", opts) do now = Keyword.fetch!(opts, :now) - last = earliest(source_query.date_range.last, now) - diff_in_days = Date.diff(source_query.date_range.first, last) - 1 + last = earliest(source_date_range.last, now) + diff_in_days = Date.diff(source_date_range.first, last) - 1 - new_first = Date.add(source_query.date_range.first, diff_in_days) + new_first = Date.add(source_date_range.first, diff_in_days) new_last = Date.add(last, diff_in_days) - range = Date.range(new_first, new_last) + comparison_date_range = + Date.range(new_first, new_last) + |> maybe_match_day_of_week(source_date_range, opts) - comparison_query = - source_query - |> Map.put(:date_range, range) - |> maybe_match_day_of_week(source_query, opts) - - {:ok, comparison_query} + {:ok, comparison_date_range} end - defp do_compare(source_query, "custom", opts) do + defp get_comparison_date_range(_source_date_range, "custom", opts) do with {:ok, from} <- opts |> Keyword.fetch!(:from) |> Date.from_iso8601(), {:ok, to} <- opts |> Keyword.fetch!(:to) |> Date.from_iso8601(), result when result in [:eq, :lt] <- Date.compare(from, to) do - {:ok, %Stats.Query{source_query | date_range: Date.range(from, to)}} + {:ok, Date.range(from, to)} else _error -> {:error, :invalid_dates} end @@ -116,24 +122,23 @@ defmodule Plausible.Stats.Comparisons do if Date.compare(a, b) in [:eq, :lt], do: a, else: b end - defp maybe_match_day_of_week(comparison_query, source_query, opts) do + defp maybe_match_day_of_week(comparison_date_range, source_date_range, opts) do if Keyword.fetch!(opts, :match_day_of_week?) do - day_to_match = Date.day_of_week(source_query.date_range.first) + day_to_match = Date.day_of_week(source_date_range.first) new_first = shift_to_nearest( day_to_match, - comparison_query.date_range.first, - source_query.date_range.first + comparison_date_range.first, + source_date_range.first ) - days_shifted = Date.diff(new_first, comparison_query.date_range.first) - new_last = Date.add(comparison_query.date_range.last, days_shifted) + days_shifted = Date.diff(new_first, comparison_date_range.first) + new_last = Date.add(comparison_date_range.last, days_shifted) - new_range = Date.range(new_first, new_last) - %Stats.Query{comparison_query | date_range: new_range} + Date.range(new_first, new_last) else - comparison_query + comparison_date_range end end diff --git a/lib/plausible/stats/datetime_range.ex b/lib/plausible/stats/datetime_range.ex new file mode 100644 index 000000000..e8e787a71 --- /dev/null +++ b/lib/plausible/stats/datetime_range.ex @@ -0,0 +1,54 @@ +defmodule Plausible.Stats.DateTimeRange do + @moduledoc """ + Defines a struct similar `Date.Range`, but with `DateTime` instead of `Date`. + + The structs should be created with the `new!/2` function. + """ + + @enforce_keys [:first, :last] + defstruct [:first, :last] + + @type t() :: %__MODULE__{ + first: %DateTime{}, + last: %DateTime{} + } + + @doc """ + Creates a `DateTimeRange` struct from the given `%Date{}` structs. + + The first datetime will become the first date at 00:00:00, and the last datetime + will become the last date at 23:59:59. Both dates will be turned into `%DateTime{}` + structs in the given timezone. + """ + def new!(%Date{} = first, %Date{} = last, timezone) do + first = + case DateTime.new(first, ~T[00:00:00], timezone) do + {:ok, datetime} -> datetime + {:gap, _just_before, just_after} -> just_after + {:ambiguous, _first_datetime, second_datetime} -> second_datetime + end + + last = + case DateTime.new(last, ~T[23:59:59], timezone) do + {:ok, datetime} -> datetime + {:gap, just_before, _just_after} -> just_before + {:ambiguous, first_datetime, _second_datetime} -> first_datetime + end + + new!(first, last) + end + + def new!(%DateTime{} = first, %DateTime{} = last) do + first = DateTime.truncate(first, :second) + last = DateTime.truncate(last, :second) + + %__MODULE__{first: first, last: last} + end + + def to_date_range(%__MODULE__{first: first, last: last}) do + first = DateTime.to_date(first) + last = DateTime.to_date(last) + + Date.range(first, last) + end +end diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.ex index f5f93e72f..6ee02bd88 100644 --- a/lib/plausible/stats/filters/query_parser.ex +++ b/lib/plausible/stats/filters/query_parser.ex @@ -1,11 +1,7 @@ defmodule Plausible.Stats.Filters.QueryParser do @moduledoc false - alias Plausible.Stats.TableDecider - alias Plausible.Stats.Filters - alias Plausible.Stats.Query - alias Plausible.Stats.Metrics - alias Plausible.Stats.JSONSchema + alias Plausible.Stats.{TableDecider, Filters, Query, Metrics, DateTimeRange, JSONSchema} @default_include %{ imports: false, @@ -13,11 +9,18 @@ defmodule Plausible.Stats.Filters.QueryParser do } def parse(site, schema_type, params, now \\ nil) when is_map(params) do + {now, date} = + if now do + {now, DateTime.shift_zone!(now, site.timezone) |> DateTime.to_date()} + else + {DateTime.utc_now(:second), today(site)} + end + with :ok <- JSONSchema.validate(schema_type, params), - {:ok, date} <- parse_date(site, Map.get(params, "date"), now), + {:ok, date} <- parse_date(site, Map.get(params, "date"), date), + {:ok, date_range} <- parse_date_range(site, Map.get(params, "date_range"), date, now), {:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])), {:ok, filters} <- parse_filters(Map.get(params, "filters", [])), - {:ok, date_range} <- parse_date_range(site, Map.get(params, "date_range"), date), {:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])), {:ok, order_by} <- parse_order_by(Map.get(params, "order_by")), {:ok, include} <- parse_include(Map.get(params, "include", %{})), @@ -110,84 +113,118 @@ defmodule Plausible.Stats.Filters.QueryParser do defp parse_clauses_list(filter), do: {:error, "Invalid filter '#{i(filter)}'"} - defp parse_date(site, nil, nil), do: {:ok, today(site)} - - defp parse_date(_site, date_string, nil) when is_binary(date_string) do + defp parse_date(_site, date_string, _date) when is_binary(date_string) do case Date.from_iso8601(date_string) do {:ok, date} -> {:ok, date} _ -> {:error, "Invalid date '#{date_string}'."} end end - defp parse_date(_site, _date_string, param_date), do: {:ok, param_date} - - defp parse_date_range(_site, "day", date) do - {:ok, Date.range(date, date)} + defp parse_date(_site, _date_string, date) do + {:ok, date} end - defp parse_date_range(_site, "7d", last) do - first = last |> Date.add(-6) - {:ok, Date.range(first, last)} + defp parse_date_range(_site, date_range, _date, now) when date_range in ["realtime", "30m"] do + duration_minutes = + case date_range do + "realtime" -> 5 + "30m" -> 30 + end + + first_datetime = DateTime.shift(now, minute: -duration_minutes) + last_datetime = DateTime.shift(now, second: 5) + + {:ok, DateTimeRange.new!(first_datetime, last_datetime)} end - defp parse_date_range(_site, "30d", last) do - first = last |> Date.add(-30) - {:ok, Date.range(first, last)} + defp parse_date_range(site, "day", date, _now) do + {:ok, DateTimeRange.new!(date, date, site.timezone)} end - defp parse_date_range(_site, "month", today) do - last = today |> Date.end_of_month() + defp parse_date_range(site, "7d", date, _now) do + first = date |> Date.add(-6) + {:ok, DateTimeRange.new!(first, date, site.timezone)} + end + + defp parse_date_range(site, "30d", date, _now) do + first = date |> Date.add(-30) + {:ok, DateTimeRange.new!(first, date, site.timezone)} + end + + defp parse_date_range(site, "month", date, _now) do + last = date |> Date.end_of_month() first = last |> Date.beginning_of_month() - {:ok, Date.range(first, last)} + {:ok, DateTimeRange.new!(first, last, site.timezone)} end - defp parse_date_range(_site, "6mo", today) do - last = today |> Date.end_of_month() + defp parse_date_range(site, "6mo", date, _now) do + last = date |> Date.end_of_month() first = last |> Date.shift(month: -5) |> Date.beginning_of_month() - {:ok, Date.range(first, last)} + {:ok, DateTimeRange.new!(first, last, site.timezone)} end - defp parse_date_range(_site, "12mo", today) do - last = today |> Date.end_of_month() + defp parse_date_range(site, "12mo", date, _now) do + last = date |> Date.end_of_month() first = last |> Date.shift(month: -11) |> Date.beginning_of_month() - {:ok, Date.range(first, last)} + {:ok, DateTimeRange.new!(first, last, site.timezone)} end - defp parse_date_range(_site, "year", today) do - last = today |> Timex.end_of_year() + defp parse_date_range(site, "year", date, _now) do + last = date |> Timex.end_of_year() first = last |> Timex.beginning_of_year() - {:ok, Date.range(first, last)} + {:ok, DateTimeRange.new!(first, last, site.timezone)} end - defp parse_date_range(site, "all", today) do - start_date = Plausible.Sites.stats_start_date(site) || today + defp parse_date_range(site, "all", date, _now) do + start_date = Plausible.Sites.stats_start_date(site) || date - {:ok, Date.range(start_date, today)} + {:ok, DateTimeRange.new!(start_date, date, site.timezone)} end - defp parse_date_range(_site, [from_date_string, to_date_string], _date) - when is_binary(from_date_string) and is_binary(to_date_string) do - with {:ok, from_date} <- Date.from_iso8601(from_date_string), - {:ok, to_date} <- Date.from_iso8601(to_date_string) do - {:ok, Date.range(from_date, to_date)} - else - _ -> {:error, "Invalid date_range '#{i([from_date_string, to_date_string])}'."} + defp parse_date_range(site, [from, to], _date, _now) + when is_binary(from) and is_binary(to) do + case date_range_from_date_strings(site, from, to) do + {:ok, date_range} -> {:ok, date_range} + {:error, _} -> date_range_from_timestamps(from, to) end end - defp parse_date_range(_site, unknown, _), + defp parse_date_range(_site, unknown, _date, _now), do: {:error, "Invalid date_range '#{i(unknown)}'."} + defp date_range_from_date_strings(site, from, to) do + with {:ok, from_date} <- Date.from_iso8601(from), + {:ok, to_date} <- Date.from_iso8601(to) do + {:ok, DateTimeRange.new!(from_date, to_date, site.timezone)} + end + end + + defp date_range_from_timestamps(from, to) do + with {:ok, from_datetime} <- datetime_from_timestamp(from), + {:ok, to_datetime} <- datetime_from_timestamp(to) do + {:ok, DateTimeRange.new!(from_datetime, to_datetime)} + else + _ -> {:error, "Invalid date_range '#{i([from, to])}'."} + end + end + + defp datetime_from_timestamp(timestamp_string) do + with [timestamp, timezone] <- String.split(timestamp_string), + {:ok, naive_datetime} <- NaiveDateTime.from_iso8601(timestamp) do + DateTime.from_naive(naive_datetime, timezone) + end + end + defp today(site), do: DateTime.now!(site.timezone) |> DateTime.to_date() defp parse_dimensions(dimensions) when is_list(dimensions) do diff --git a/lib/plausible/stats/goal_suggestions.ex b/lib/plausible/stats/goal_suggestions.ex index 4ff23bc8e..d3192bfb8 100644 --- a/lib/plausible/stats/goal_suggestions.ex +++ b/lib/plausible/stats/goal_suggestions.ex @@ -61,7 +61,9 @@ defmodule Plausible.Stats.GoalSuggestions do from(i in "imported_custom_events", where: i.site_id == ^site.id, where: i.import_id in ^site.complete_import_ids, - where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last, + where: + i.date >= ^DateTime.to_naive(query.date_range.first) and + i.date <= ^DateTime.to_naive(query.date_range.last), where: i.visitors > 0, where: fragment("? ilike ?", i.name, ^matches), where: fragment("trim(?)", i.name) != "", diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.ex index 5bc46d07e..23bfebda4 100644 --- a/lib/plausible/stats/imported/base.ex +++ b/lib/plausible/stats/imported/base.ex @@ -6,7 +6,7 @@ defmodule Plausible.Stats.Imported.Base do import Ecto.Query alias Plausible.Imported - alias Plausible.Stats.{Filters, Query, SQL} + alias Plausible.Stats.{Filters, Query, SQL, DateTimeRange} @property_to_table_mappings %{ "visit:source" => "imported_sources", @@ -71,7 +71,7 @@ defmodule Plausible.Stats.Imported.Base do def query_imported(table, site, query) do import_ids = site.complete_import_ids - %{first: date_from, last: date_to} = query.date_range + %{first: date_from, last: date_to} = DateTimeRange.to_date_range(query.date_range) from(i in table, where: i.site_id == ^site.id, diff --git a/lib/plausible/stats/imported/sql/builder.ex b/lib/plausible/stats/imported/sql/builder.ex index 96a846121..76a8b011d 100644 --- a/lib/plausible/stats/imported/sql/builder.ex +++ b/lib/plausible/stats/imported/sql/builder.ex @@ -293,7 +293,7 @@ defmodule Plausible.Stats.Imported.SQL.Builder do defp select_group_fields(q, "time:week", key, query) do select_merge_as(q, [i], %{ - key => weekstart_not_before(i.date, ^query.date_range.first) + key => weekstart_not_before(i.date, ^DateTime.to_naive(query.date_range.first)) }) end diff --git a/lib/plausible/stats/interval.ex b/lib/plausible/stats/interval.ex index eb501adf7..fc3324d4c 100644 --- a/lib/plausible/stats/interval.ex +++ b/lib/plausible/stats/interval.ex @@ -7,6 +7,8 @@ defmodule Plausible.Stats.Interval do `week`, and `month`. """ + alias Plausible.Stats.DateTimeRange + @type t() :: String.t() @type(opt() :: {:site, Plausible.Site.t()} | {:from, Date.t()}, {:to, Date.t()}) @type opts :: list(opt()) @@ -27,18 +29,20 @@ defmodule Plausible.Stats.Interval do """ def default_for_period(period) do case period do - "realtime" -> "minute" + period when period in ["realtime", "30m"] -> "minute" "day" -> "hour" period when period in ["custom", "7d", "30d", "month"] -> "day" period when period in ["6mo", "12mo", "year"] -> "month" end end - @spec default_for_date_range(Date.Range.t()) :: t() + @spec default_for_date_range(DateTimeRange.t()) :: t() @doc """ - Returns the suggested interval for the given `Date.Range` struct. + Returns the suggested interval for the given `DateTimeRange` struct. """ - def default_for_date_range(%Date.Range{first: first, last: last}) do + def default_for_date_range(%DateTimeRange{} = date_range) do + %Date.Range{first: first, last: last} = DateTimeRange.to_date_range(date_range) + cond do Timex.diff(last, first, :months) > 0 -> "month" diff --git a/lib/plausible/stats/json_schema.ex b/lib/plausible/stats/json_schema.ex index 08c22467a..c1f9beede 100644 --- a/lib/plausible/stats/json_schema.ex +++ b/lib/plausible/stats/json_schema.ex @@ -27,6 +27,13 @@ defmodule Plausible.Stats.JSONSchema do |> JSONPointer.add!("#/definitions/metric/oneOf/0", %{ "const" => "time_on_page" }) + |> JSONPointer.add!("#/definitions/date_range/oneOf/0", %{ + "const" => "30m" + }) + |> JSONPointer.add!("#/definitions/date_range/oneOf/0", %{ + "const" => "realtime" + }) + |> JSONPointer.add!("#/properties/date", %{"type" => "string"}) |> ExJsonSchema.Schema.resolve() def validate(schema_type, params) do diff --git a/lib/plausible/stats/legacy/legacy_query_builder.ex b/lib/plausible/stats/legacy/legacy_query_builder.ex index 3abd8194c..cd87b8670 100644 --- a/lib/plausible/stats/legacy/legacy_query_builder.ex +++ b/lib/plausible/stats/legacy/legacy_query_builder.ex @@ -3,10 +3,10 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do use Plausible - alias Plausible.Stats.{Filters, Interval, Query} + alias Plausible.Stats.{Filters, Interval, Query, DateTimeRange} def from(site, params, debug_metadata) do - now = NaiveDateTime.utc_now(:second) + now = DateTime.utc_now(:second) query = Query @@ -37,46 +37,50 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do struct!(query, preloaded_goals: goals) end - defp put_period(query, site, %{"period" => "realtime"}) do - date = today(site.timezone) + defp put_period(%Query{now: now} = query, _site, %{"period" => period}) + when period in ["realtime", "30m"] do + duration_minutes = + case period do + "realtime" -> 5 + "30m" -> 30 + end - struct!(query, period: "realtime", date_range: Date.range(date, date)) + first_datetime = DateTime.shift(now, minute: -duration_minutes) + last_datetime = DateTime.shift(now, second: 5) + + struct!(query, period: period, date_range: DateTimeRange.new!(first_datetime, last_datetime)) end defp put_period(query, site, %{"period" => "day"} = params) do date = parse_single_date(site.timezone, params) + datetime_range = DateTimeRange.new!(date, date, site.timezone) - struct!(query, period: "day", date_range: Date.range(date, date)) + struct!(query, period: "day", date_range: datetime_range) end defp put_period(query, site, %{"period" => "7d"} = params) do end_date = parse_single_date(site.timezone, params) start_date = end_date |> Date.shift(day: -6) + datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone) - struct!( - query, - period: "7d", - date_range: Date.range(start_date, end_date) - ) + struct!(query, period: "7d", date_range: datetime_range) end defp put_period(query, site, %{"period" => "30d"} = params) do end_date = parse_single_date(site.timezone, params) start_date = end_date |> Date.shift(day: -30) + datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone) - struct!(query, period: "30d", date_range: Date.range(start_date, end_date)) + struct!(query, period: "30d", date_range: datetime_range) end defp put_period(query, site, %{"period" => "month"} = params) do date = parse_single_date(site.timezone, params) - start_date = Timex.beginning_of_month(date) end_date = Timex.end_of_month(date) + datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone) - struct!(query, - period: "month", - date_range: Date.range(start_date, end_date) - ) + struct!(query, period: "month", date_range: datetime_range) end defp put_period(query, site, %{"period" => "6mo"} = params) do @@ -88,10 +92,9 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do Date.shift(end_date, month: -5) |> Timex.beginning_of_month() - struct!(query, - period: "6mo", - date_range: Date.range(start_date, end_date) - ) + datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone) + + struct!(query, period: "6mo", date_range: datetime_range) end defp put_period(query, site, %{"period" => "12mo"} = params) do @@ -103,10 +106,9 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do Date.shift(end_date, month: -11) |> Timex.beginning_of_month() - struct!(query, - period: "12mo", - date_range: Date.range(start_date, end_date) - ) + datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone) + + struct!(query, period: "12mo", date_range: datetime_range) end defp put_period(query, site, %{"period" => "year"} = params) do @@ -115,21 +117,17 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do |> Timex.end_of_year() start_date = Timex.beginning_of_year(end_date) + datetime_range = DateTimeRange.new!(start_date, end_date, site.timezone) - struct!(query, - period: "year", - date_range: Date.range(start_date, end_date) - ) + struct!(query, period: "year", date_range: datetime_range) end defp put_period(query, site, %{"period" => "all"}) do - now = today(site.timezone) - start_date = Plausible.Sites.stats_start_date(site) || now + today = today(site.timezone) + start_date = Plausible.Sites.stats_start_date(site) || today + datetime_range = DateTimeRange.new!(start_date, today, site.timezone) - struct!(query, - period: "all", - date_range: Date.range(start_date, now) - ) + struct!(query, period: "all", date_range: datetime_range) end defp put_period(query, site, %{"period" => "custom", "from" => from, "to" => to} = params) do @@ -141,15 +139,13 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do put_period(query, site, new_params) end - defp put_period(query, _site, %{"period" => "custom", "date" => date}) do + defp put_period(query, site, %{"period" => "custom", "date" => date}) do [from, to] = String.split(date, ",") from_date = Date.from_iso8601!(String.trim(from)) to_date = Date.from_iso8601!(String.trim(to)) + datetime_range = DateTimeRange.new!(from_date, to_date, site.timezone) - struct!(query, - period: "custom", - date_range: Date.range(from_date, to_date) - ) + struct!(query, period: "custom", date_range: datetime_range) end defp put_period(query, site, params) do diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index cbbf6ae31..71f5f200e 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -35,7 +35,7 @@ defmodule Plausible.Stats.Query do struct!(__MODULE__, Map.to_list(query_data)) |> put_imported_opts(site, %{}) |> put_experimental_reduced_joins(site, params) - |> struct!(v2: true, now: NaiveDateTime.utc_now(:second), debug_metadata: debug_metadata) + |> struct!(v2: true, now: DateTime.utc_now(:second), debug_metadata: debug_metadata) {:ok, query} end @@ -142,9 +142,9 @@ defmodule Plausible.Stats.Query do def ensure_include_imported(query, requested?) do cond do is_nil(query.latest_import_end_date) -> {:error, :no_imported_data} + query.period in ["realtime", "30m"] -> {:error, :unsupported_query} Date.after?(query.date_range.first, query.latest_import_end_date) -> {:error, :out_of_range} not Imported.schema_supports_query?(query) -> {:error, :unsupported_query} - query.period == "realtime" -> {:error, :unsupported_query} not requested? -> {:error, :not_requested} true -> :ok end diff --git a/lib/plausible/stats/query_optimizer.ex b/lib/plausible/stats/query_optimizer.ex index 5f3bf706f..f5e9b272b 100644 --- a/lib/plausible/stats/query_optimizer.ex +++ b/lib/plausible/stats/query_optimizer.ex @@ -4,7 +4,7 @@ defmodule Plausible.Stats.QueryOptimizer do """ use Plausible - alias Plausible.Stats.{Query, TableDecider, Util} + alias Plausible.Stats.{Query, TableDecider, Util, DateTimeRange} @doc """ This module manipulates an existing query, updating it according to business logic. @@ -61,7 +61,7 @@ defmodule Plausible.Stats.QueryOptimizer do defp update_group_by_time( %Query{ - date_range: %Date.Range{first: first, last: last} + date_range: %DateTimeRange{first: first, last: last} } = query ) do dimensions = diff --git a/lib/plausible/stats/query_result.ex b/lib/plausible/stats/query_result.ex index 3a9ce3c56..22ecc654a 100644 --- a/lib/plausible/stats/query_result.ex +++ b/lib/plausible/stats/query_result.ex @@ -7,8 +7,7 @@ defmodule Plausible.Stats.QueryResult do produced by Jason.encode(query_result) is ordered. """ - alias Plausible.Stats.Util - alias Plausible.Stats.Filters + alias Plausible.Stats.{Util, Filters} defstruct results: [], meta: %{}, @@ -32,7 +31,10 @@ defmodule Plausible.Stats.QueryResult do Jason.OrderedObject.new( site_id: site.domain, metrics: query.metrics, - date_range: [query.date_range.first, query.date_range.last], + date_range: [ + to_iso_8601_with_timezone(query.date_range.first), + to_iso_8601_with_timezone(query.date_range.last) + ], filters: query.filters, dimensions: query.dimensions, order_by: query.order_by |> Enum.map(&Tuple.to_list/1), @@ -79,6 +81,15 @@ defmodule Plausible.Stats.QueryResult do |> Enum.reject(fn {_, value} -> is_nil(value) end) |> Enum.into(%{}) end + + defp to_iso_8601_with_timezone(%DateTime{time_zone: timezone} = datetime) do + naive_iso8601 = + datetime + |> DateTime.to_naive() + |> NaiveDateTime.to_iso8601() + + naive_iso8601 <> " " <> timezone + end end defimpl Jason.Encoder, for: Plausible.Stats.QueryResult do diff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.ex index 4638ea7fc..cdec499d1 100644 --- a/lib/plausible/stats/sql/expression.ex +++ b/lib/plausible/stats/sql/expression.ex @@ -50,7 +50,7 @@ defmodule Plausible.Stats.SQL.Expression do key => weekstart_not_before( to_timezone(t.timestamp, ^query.timezone), - ^query.date_range.first + ^DateTime.to_naive(query.date_range.first) ) }) end @@ -83,27 +83,6 @@ defmodule Plausible.Stats.SQL.Expression do }) end - # :NOTE: This is not exposed in Query APIv2 - def select_dimension(q, key, "time:minute", :sessions, %Query{ - period: "30m" - }) do - select_merge_as(q, [s], %{ - key => - fragment( - "arrayJoin(range(dateDiff('minute', now(), ?), dateDiff('minute', now(), ?) + 1))", - s.start, - s.timestamp - ) - }) - end - - # :NOTE: This is not exposed in Query APIv2 - def select_dimension(q, key, "time:minute", _table, %Query{period: "30m"}) do - select_merge_as(q, [t], %{ - key => fragment("dateDiff('minute', now(), ?)", t.timestamp) - }) - end - # :NOTE: This is not exposed in Query APIv2 def select_dimension(q, key, "time:minute", :sessions, query) do q diff --git a/lib/plausible/stats/sql/where_builder.ex b/lib/plausible/stats/sql/where_builder.ex index 6cda55cf1..51a7b7090 100644 --- a/lib/plausible/stats/sql/where_builder.ex +++ b/lib/plausible/stats/sql/where_builder.ex @@ -48,7 +48,7 @@ defmodule Plausible.Stats.SQL.WhereBuilder do dynamic( [e], - e.site_id == ^site.id and e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime + e.site_id == ^site.id and e.timestamp >= ^first_datetime and e.timestamp <= ^last_datetime ) end @@ -71,7 +71,7 @@ defmodule Plausible.Stats.SQL.WhereBuilder do s.site_id == ^site.id and s.start >= ^NaiveDateTime.add(first_datetime, -7, :day) and s.timestamp >= ^first_datetime and - s.start < ^last_datetime + s.start <= ^last_datetime ) end diff --git a/lib/plausible/stats/time.ex b/lib/plausible/stats/time.ex index f87c40e4e..4b9b4b6a5 100644 --- a/lib/plausible/stats/time.ex +++ b/lib/plausible/stats/time.ex @@ -3,52 +3,20 @@ defmodule Plausible.Stats.Time do Collection of functions to work with time in queries. """ - alias Plausible.Stats.Query - alias Plausible.Timezones - - def utc_boundaries(%Query{period: "realtime", now: now}, site) do - last_datetime = - now - |> NaiveDateTime.shift(second: 5) - |> NaiveDateTime.truncate(:second) - - first_datetime = - now - |> NaiveDateTime.shift(minute: -5) - |> NaiveDateTime.truncate(:second) - |> beginning_of_time(site.native_stats_start_at) - - {first_datetime, last_datetime} - end - - def utc_boundaries(%Query{period: "30m", now: now}, site) do - last_datetime = - now - |> NaiveDateTime.shift(second: 5) - |> NaiveDateTime.truncate(:second) - - first_datetime = - now - |> NaiveDateTime.shift(minute: -30) - |> NaiveDateTime.truncate(:second) - |> beginning_of_time(site.native_stats_start_at) - - {first_datetime, last_datetime} - end + alias Plausible.Stats.{Query, DateTimeRange} def utc_boundaries(%Query{date_range: date_range}, site) do - {:ok, first} = NaiveDateTime.new(date_range.first, ~T[00:00:00]) + %DateTimeRange{first: first, last: last} = date_range - first_datetime = + first = first - |> Timezones.to_utc_datetime(site.timezone) + |> DateTime.shift_zone!("Etc/UTC") + |> DateTime.to_naive() |> beginning_of_time(site.native_stats_start_at) - {:ok, last} = NaiveDateTime.new(date_range.last |> Date.shift(day: 1), ~T[00:00:00]) + last = DateTime.shift_zone!(last, "Etc/UTC") |> DateTime.to_naive() - last_datetime = Timezones.to_utc_datetime(last, site.timezone) - - {first_datetime, last_datetime} + {first, last} end defp beginning_of_time(candidate, native_stats_start_at) do @@ -61,7 +29,7 @@ defmodule Plausible.Stats.Time do def format_datetime(%Date{} = date), do: Date.to_string(date) - def format_datetime(%DateTime{} = datetime), + def format_datetime(%mod{} = datetime) when mod in [NaiveDateTime, DateTime], do: Calendar.strftime(datetime, "%Y-%m-%d %H:%M:%S") # Realtime graphs return numbers @@ -79,15 +47,17 @@ defmodule Plausible.Stats.Time do end defp time_labels_for_dimension("time:month", query) do + date_range = DateTimeRange.to_date_range(query.date_range) + n_buckets = Timex.diff( - query.date_range.last, - Date.beginning_of_month(query.date_range.first), + date_range.last, + Date.beginning_of_month(date_range.first), :months ) Enum.map(n_buckets..0, fn shift -> - query.date_range.last + date_range.last |> Date.beginning_of_month() |> Date.shift(month: -shift) |> format_datetime() @@ -95,15 +65,17 @@ defmodule Plausible.Stats.Time do end defp time_labels_for_dimension("time:week", query) do + date_range = DateTimeRange.to_date_range(query.date_range) + n_buckets = Timex.diff( - query.date_range.last, - Date.beginning_of_week(query.date_range.first), + date_range.last, + Date.beginning_of_week(date_range.first), :weeks ) Enum.map(0..n_buckets, fn shift -> - query.date_range.first + date_range.first |> Date.shift(week: shift) |> date_or_weekstart(query) |> format_datetime() @@ -112,59 +84,42 @@ defmodule Plausible.Stats.Time do defp time_labels_for_dimension("time:day", query) do query.date_range + |> DateTimeRange.to_date_range() |> Enum.into([]) |> Enum.map(&format_datetime/1) end - @full_day_in_hours 23 defp time_labels_for_dimension("time:hour", query) do - n_buckets = - if query.date_range.first == query.date_range.last do - @full_day_in_hours - else - end_time = - query.date_range.last - |> Timex.to_datetime() - |> Timex.end_of_day() - - Timex.diff(end_time, query.date_range.first, :hours) - end + n_buckets = DateTime.diff(query.date_range.last, query.date_range.first, :hour) Enum.map(0..n_buckets, fn step -> query.date_range.first - |> Timex.to_datetime() - |> DateTime.shift(hour: step) - |> DateTime.truncate(:second) + |> DateTime.to_naive() + |> NaiveDateTime.shift(hour: step) |> format_datetime() end) end - # Only supported in dashboards not via API - defp time_labels_for_dimension("time:minute", %Query{period: "30m"}) do - Enum.into(-30..-1, []) - end - - @full_day_in_minutes 24 * 60 - 1 defp time_labels_for_dimension("time:minute", query) do - n_buckets = - if query.date_range.first == query.date_range.last do - @full_day_in_minutes - else - Timex.diff(query.date_range.last, query.date_range.first, :minutes) - end + first_datetime = Map.put(query.date_range.first, :second, 0) - Enum.map(0..n_buckets, fn step -> - query.date_range.first - |> Timex.to_datetime() - |> DateTime.shift(minute: step) - |> format_datetime() + first_datetime + |> Stream.iterate(fn datetime -> DateTime.shift(datetime, minute: 1) end) + |> Enum.take_while(fn datetime -> + current_minute = Map.put(query.now, :second, 0) + + DateTime.before?(datetime, query.date_range.last) && + DateTime.before?(datetime, current_minute) end) + |> Enum.map(&format_datetime/1) end - defp date_or_weekstart(date, query) do - weekstart = Timex.beginning_of_week(date) + def date_or_weekstart(date, query) do + weekstart = Date.beginning_of_week(date) - if Enum.member?(query.date_range, weekstart) do + date_range = DateTimeRange.to_date_range(query.date_range) + + if Enum.member?(date_range, weekstart) do weekstart else date diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex index 8b3ddbcc3..8c6bf8512 100644 --- a/lib/plausible/stats/timeseries.ex +++ b/lib/plausible/stats/timeseries.ex @@ -69,6 +69,7 @@ defmodule Plausible.Stats.Timeseries do ) |> cast_revenue_metrics_to_money(currency) end) + |> transform_realtime_labels(query) end defp empty_row(date, metrics) do @@ -102,6 +103,13 @@ defmodule Plausible.Stats.Timeseries do end) end + defp transform_realtime_labels(results, %Query{period: "30m"}) do + Enum.with_index(results) + |> Enum.map(fn {entry, index} -> %{entry | date: -30 + index} end) + end + + defp transform_realtime_labels(results, _query), do: results + on_ee do defp cast_revenue_metrics_to_money(results, revenue_goals) do Plausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(results, revenue_goals) diff --git a/lib/plausible/timezones.ex b/lib/plausible/timezones.ex index 9aa030e3d..bcd161ad4 100644 --- a/lib/plausible/timezones.ex +++ b/lib/plausible/timezones.ex @@ -6,20 +6,6 @@ defmodule Plausible.Timezones do |> Enum.sort_by(& &1[:offset], :desc) end - @spec to_utc_datetime(NaiveDateTime.t(), String.t()) :: DateTime.t() - def to_utc_datetime(naive_date_time, timezone) do - case Timex.to_datetime(naive_date_time, timezone) do - %DateTime{} = tz_dt -> - Timex.Timezone.convert(tz_dt, "UTC") - - %Timex.AmbiguousDateTime{after: after_dt} -> - Timex.Timezone.convert(after_dt, "UTC") - - {:error, {:could_not_resolve_timezone, _, _, _}} -> - Timex.Timezone.convert(naive_date_time, "UTC") - end - end - @spec to_date_in_timezone(Date.t() | NaiveDateTime.t() | DateTime.t(), String.t()) :: Date.t() def to_date_in_timezone(dt, timezone) do to_datetime_in_timezone(dt, timezone) |> Timex.to_date() diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 4db14ec22..687e28825 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -5,7 +5,7 @@ defmodule PlausibleWeb.Api.StatsController do use PlausibleWeb.Plugs.ErrorHandler alias Plausible.Stats - alias Plausible.Stats.{Query, Comparisons} + alias Plausible.Stats.{Query, Comparisons, Time, DateTimeRange} alias Plausible.Stats.Filters.LegacyDashboardFilterParser alias PlausibleWeb.Api.Helpers, as: H @@ -104,16 +104,10 @@ defmodule PlausibleWeb.Api.StatsController do with {:ok, dates} <- parse_date_params(params), :ok <- validate_interval(params), :ok <- validate_interval_granularity(site, params, dates), + params <- realtime_period_to_30m(params), query = Query.from(site, params, debug_metadata(conn)), {:ok, metric} <- parse_and_validate_graph_metric(params, query) do - timeseries_query = - if query.period == "realtime" do - %Query{query | period: "30m"} - else - query - end - - timeseries_result = Stats.timeseries(site, timeseries_query, [metric]) + timeseries_result = Stats.timeseries(site, query, [metric]) comparison_opts = parse_comparison_opts(params) @@ -173,10 +167,12 @@ defmodule PlausibleWeb.Api.StatsController do end defp build_full_intervals(%{interval: "week", date_range: date_range}, labels) do + date_range = DateTimeRange.to_date_range(date_range) build_intervals(labels, date_range, &Timex.beginning_of_week/1, &Timex.end_of_week/1) end defp build_full_intervals(%{interval: "month", date_range: date_range}, labels) do + date_range = DateTimeRange.to_date_range(date_range) build_intervals(labels, date_range, &Timex.beginning_of_month/1, &Timex.end_of_month/1) end @@ -205,6 +201,8 @@ defmodule PlausibleWeb.Api.StatsController do def top_stats(conn, params) do site = conn.assigns[:site] + params = realtime_period_to_30m(params) + query = Query.from(site, params, debug_metadata(conn)) comparison_opts = parse_comparison_opts(params) @@ -224,14 +222,14 @@ defmodule PlausibleWeb.Api.StatsController do with_imported_switch: with_imported_switch_info(query, comparison_query), includes_imported: includes_imported?(query, comparison_query), imports_exist: site.complete_import_ids != [], - comparing_from: comparison_query && comparison_query.date_range.first, - comparing_to: comparison_query && comparison_query.date_range.last, - from: query.date_range.first, - to: query.date_range.last + comparing_from: comparison_query && DateTime.to_date(comparison_query.date_range.first), + comparing_to: comparison_query && DateTime.to_date(comparison_query.date_range.last), + from: DateTime.to_date(query.date_range.first), + to: DateTime.to_date(query.date_range.last) }) end - defp with_imported_switch_info(%Query{period: "realtime"}, _) do + defp with_imported_switch_info(%Query{period: "30m"}, _) do %{visible: false, togglable: false, tooltip_msg: nil} end @@ -284,7 +282,7 @@ defmodule PlausibleWeb.Api.StatsController do current_date = Timex.now(site.timezone) |> Timex.to_date() - |> date_or_weekstart(query) + |> Time.date_or_weekstart(query) |> Date.to_string() Enum.find_index(dates, &(&1 == current_date)) @@ -307,24 +305,14 @@ defmodule PlausibleWeb.Api.StatsController do end end - defp date_or_weekstart(date, query) do - weekstart = Timex.beginning_of_week(date) - - if Enum.member?(query.date_range, weekstart) do - weekstart - else - date - end - end - defp fetch_top_stats(site, query, comparison_query) do goal_filter = Query.get_filter(query, "event:goal") cond do - query.period == "realtime" && goal_filter -> + query.period == "30m" && goal_filter -> fetch_goal_realtime_top_stats(site, query, comparison_query) - query.period == "realtime" -> + query.period == "30m" -> fetch_realtime_top_stats(site, query, comparison_query) goal_filter -> @@ -336,12 +324,10 @@ defmodule PlausibleWeb.Api.StatsController do end defp fetch_goal_realtime_top_stats(site, query, _comparison_query) do - query_30m = %Query{query | period: "30m"} - %{ visitors: %{value: unique_conversions}, events: %{value: total_conversions} - } = Stats.aggregate(site, query_30m, [:visitors, :events]) + } = Stats.aggregate(site, query, [:visitors, :events]) stats = [ %{ @@ -364,12 +350,10 @@ defmodule PlausibleWeb.Api.StatsController do end defp fetch_realtime_top_stats(site, query, _comparison_query) do - query_30m = %Query{query | period: "30m"} - %{ visitors: %{value: visitors}, pageviews: %{value: pageviews} - } = Stats.aggregate(site, query_30m, [:visitors, :pageviews]) + } = Stats.aggregate(site, query, [:visitors, :pageviews]) stats = [ %{ @@ -1270,15 +1254,13 @@ defmodule PlausibleWeb.Api.StatsController do def conversions(conn, params) do pagination = parse_pagination(params) site = Plausible.Repo.preload(conn.assigns.site, :goals) - params = Map.put(params, "property", "event:goal") - query = Query.from(site, params, debug_metadata(conn)) - query = - if query.period == "realtime" do - %Query{query | period: "30m"} - else - query - end + params = + params + |> realtime_period_to_30m() + |> Map.put("property", "event:goal") + + query = Query.from(site, params, debug_metadata(conn)) metrics = [:visitors, :events, :conversion_rate] ++ @revenue_metrics @@ -1583,4 +1565,12 @@ defmodule PlausibleWeb.Api.StatsController do Map.put(row, :name, name) end + + defp realtime_period_to_30m(%{"period" => _} = params) do + Map.update!(params, "period", fn period -> + if period == "realtime", do: "30m", else: period + end) + end + + defp realtime_period_to_30m(params), do: params end diff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.json index 007575ae8..8e62b288e 100644 --- a/priv/json-schemas/query-api-schema.json +++ b/priv/json-schemas/query-api-schema.json @@ -99,13 +99,17 @@ "type": "array", "items": { "type": "string", - "pattern": "^\\d{4}-\\d{2}-\\d{2}$" + "pattern": "^\\d{4}-\\d{2}-\\d{2}(?:T\\d{2}:\\d{2}:\\d{2}\\s[A-Za-z/_]+)?$" }, - "markdownDescription": "Start and end dates of custom date range. Both dates are in format `YYYY-MM-DD`", + "markdownDescription": "A list of two elements to determine the query date range. Both elements should have the same format - either `YYYY-MM-DD` or `YYYY-MM-DDThh:mm:ss `", "examples": [ [ "2024-01-01", "2024-01-31" + ], + [ + "2024-01-01T00:00:00 Europe/Tallinn", + "2024-01-01T12:00:00 Europe/Tallinn" ] ], "minItems": 2, diff --git a/test/plausible/google/api_test.exs b/test/plausible/google/api_test.exs index cd020ca5e..ef6a96d93 100644 --- a/test/plausible/google/api_test.exs +++ b/test/plausible/google/api_test.exs @@ -3,6 +3,7 @@ defmodule Plausible.Google.APITest do use Plausible.Test.Support.HTTPMocker alias Plausible.Google + alias Plausible.Stats.Query import ExUnit.CaptureLog import Mox @@ -41,7 +42,8 @@ defmodule Plausible.Google.APITest do end ) - query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])} + query = + Query.from(site, %{"period" => "custom", "from" => "2022-01-01", "to" => "2022-01-05"}) assert {:error, "google_auth_error"} = Google.API.fetch_stats(site, query, {5, 0}, "") end @@ -58,7 +60,8 @@ defmodule Plausible.Google.APITest do end ) - query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])} + query = + Query.from(site, %{"period" => "custom", "from" => "2022-01-01", "to" => "2022-01-05"}) assert {:error, "some_error"} = Google.API.fetch_stats(site, query, {5, 0}, "") end @@ -75,7 +78,8 @@ defmodule Plausible.Google.APITest do end ) - query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])} + query = + Query.from(site, %{"period" => "custom", "from" => "2022-01-01", "to" => "2022-01-05"}) log = capture_log(fn -> @@ -100,7 +104,8 @@ defmodule Plausible.Google.APITest do expires: NaiveDateTime.add(NaiveDateTime.utc_now(), -3600) ) - query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])} + query = + Query.from(site, %{"period" => "custom", "from" => "2022-01-01", "to" => "2022-01-05"}) assert {:error, "invalid_grant"} = Google.API.fetch_stats(site, query, 5, "") end @@ -126,7 +131,8 @@ defmodule Plausible.Google.APITest do test "returns name and visitor count", %{site: site} do mock_http_with("google_search_console.json") - query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])} + query = + Query.from(site, %{"period" => "custom", "from" => "2022-01-01", "to" => "2022-01-05"}) assert {:ok, [ diff --git a/test/plausible/stats/comparisons_test.exs b/test/plausible/stats/comparisons_test.exs index 9f72a35f9..7d2504c6a 100644 --- a/test/plausible/stats/comparisons_test.exs +++ b/test/plausible/stats/comparisons_test.exs @@ -11,8 +11,11 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now) - assert comparison.date_range.first == ~D[2023-02-27] - assert comparison.date_range.last == ~D[2023-02-28] + assert comparison.date_range.first == + DateTime.new!(~D[2023-02-27], ~T[00:00:00], site.timezone) + + assert comparison.date_range.last == + DateTime.new!(~D[2023-02-28], ~T[23:59:59], site.timezone) end test "shifts back this month period when it's the first day of the month and mode is previous_period" do @@ -22,8 +25,11 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now) - assert comparison.date_range.first == ~D[2023-02-28] - assert comparison.date_range.last == ~D[2023-02-28] + assert comparison.date_range.first == + DateTime.new!(~D[2023-02-28], ~T[00:00:00], site.timezone) + + assert comparison.date_range.last == + DateTime.new!(~D[2023-02-28], ~T[23:59:59], site.timezone) end test "matches the day of the week when nearest day is original query start date and mode is previous_period" do @@ -34,8 +40,11 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now, match_day_of_week?: true) - assert comparison.date_range.first == ~D[2023-02-22] - assert comparison.date_range.last == ~D[2023-02-23] + assert comparison.date_range.first == + DateTime.new!(~D[2023-02-22], ~T[00:00:00], site.timezone) + + assert comparison.date_range.last == + DateTime.new!(~D[2023-02-23], ~T[23:59:59], site.timezone) end end @@ -47,8 +56,11 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now) - assert comparison.date_range.first == ~D[2023-01-04] - assert comparison.date_range.last == ~D[2023-01-31] + assert comparison.date_range.first == + DateTime.new!(~D[2023-01-04], ~T[00:00:00], site.timezone) + + assert comparison.date_range.last == + DateTime.new!(~D[2023-01-31], ~T[23:59:59], site.timezone) end test "shifts back the full month when mode is year_over_year" do @@ -58,8 +70,11 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "year_over_year", now: now) - assert comparison.date_range.first == ~D[2022-02-01] - assert comparison.date_range.last == ~D[2022-02-28] + assert comparison.date_range.first == + DateTime.new!(~D[2022-02-01], ~T[00:00:00], site.timezone) + + assert comparison.date_range.last == + DateTime.new!(~D[2022-02-28], ~T[23:59:59], site.timezone) end test "shifts back whole month plus one day when mode is year_over_year and a leap year" do @@ -69,8 +84,11 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "year_over_year", now: now) - assert comparison.date_range.first == ~D[2019-02-01] - assert comparison.date_range.last == ~D[2019-03-01] + assert comparison.date_range.first == + DateTime.new!(~D[2019-02-01], ~T[00:00:00], site.timezone) + + assert comparison.date_range.last == + DateTime.new!(~D[2019-03-01], ~T[23:59:59], site.timezone) end test "matches the day of the week when mode is previous_period keeping the same day" do @@ -81,8 +99,11 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now, match_day_of_week?: true) - assert comparison.date_range.first == ~D[2023-01-04] - assert comparison.date_range.last == ~D[2023-01-31] + assert comparison.date_range.first == + DateTime.new!(~D[2023-01-04], ~T[00:00:00], site.timezone) + + assert comparison.date_range.last == + DateTime.new!(~D[2023-01-31], ~T[23:59:59], site.timezone) end test "matches the day of the week when mode is previous_period" do @@ -93,8 +114,11 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now, match_day_of_week?: true) - assert comparison.date_range.first == ~D[2022-12-04] - assert comparison.date_range.last == ~D[2023-01-03] + assert comparison.date_range.first == + DateTime.new!(~D[2022-12-04], ~T[00:00:00], site.timezone) + + assert comparison.date_range.last == + DateTime.new!(~D[2023-01-03], ~T[23:59:59], site.timezone) end end @@ -106,8 +130,11 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "previous_period", now: now) - assert comparison.date_range.first == ~D[2022-11-02] - assert comparison.date_range.last == ~D[2022-12-31] + assert comparison.date_range.first == + DateTime.new!(~D[2022-11-02], ~T[00:00:00], site.timezone) + + assert comparison.date_range.last == + DateTime.new!(~D[2022-12-31], ~T[23:59:59], site.timezone) end test "shifts back by the same number of days when mode is year_over_year" do @@ -117,8 +144,11 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "year_over_year", now: now) - assert comparison.date_range.first == ~D[2022-01-01] - assert comparison.date_range.last == ~D[2022-03-01] + assert comparison.date_range.first == + DateTime.new!(~D[2022-01-01], ~T[00:00:00], site.timezone) + + assert comparison.date_range.last == + DateTime.new!(~D[2022-03-01], ~T[23:59:59], site.timezone) end test "matches the day of the week when mode is year_over_year" do @@ -129,8 +159,11 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "year_over_year", now: now, match_day_of_week?: true) - assert comparison.date_range.first == ~D[2022-01-02] - assert comparison.date_range.last == ~D[2022-03-02] + assert comparison.date_range.first == + DateTime.new!(~D[2022-01-02], ~T[00:00:00], site.timezone) + + assert comparison.date_range.last == + DateTime.new!(~D[2022-03-02], ~T[23:59:59], site.timezone) end end @@ -141,8 +174,11 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "year_over_year") - assert comparison.date_range.first == ~D[2021-01-01] - assert comparison.date_range.last == ~D[2021-12-31] + assert comparison.date_range.first == + DateTime.new!(~D[2021-01-01], ~T[00:00:00], site.timezone) + + assert comparison.date_range.last == + DateTime.new!(~D[2021-12-31], ~T[23:59:59], site.timezone) end test "shifts back a whole year when mode is previous_period" do @@ -151,8 +187,11 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "previous_period") - assert comparison.date_range.first == ~D[2021-01-01] - assert comparison.date_range.last == ~D[2021-12-31] + assert comparison.date_range.first == + DateTime.new!(~D[2021-01-01], ~T[00:00:00], site.timezone) + + assert comparison.date_range.last == + DateTime.new!(~D[2021-12-31], ~T[23:59:59], site.timezone) end end @@ -163,8 +202,11 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "previous_period") - assert comparison.date_range.first == ~D[2022-12-25] - assert comparison.date_range.last == ~D[2022-12-31] + assert comparison.date_range.first == + DateTime.new!(~D[2022-12-25], ~T[00:00:00], site.timezone) + + assert comparison.date_range.last == + DateTime.new!(~D[2022-12-31], ~T[23:59:59], site.timezone) end test "shifts back to last year when mode is year_over_year" do @@ -173,8 +215,11 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "year_over_year") - assert comparison.date_range.first == ~D[2022-01-01] - assert comparison.date_range.last == ~D[2022-01-07] + assert comparison.date_range.first == + DateTime.new!(~D[2022-01-01], ~T[00:00:00], site.timezone) + + assert comparison.date_range.last == + DateTime.new!(~D[2022-01-07], ~T[23:59:59], site.timezone) end end @@ -186,8 +231,11 @@ defmodule Plausible.Stats.ComparisonsTest do {:ok, comparison} = Comparisons.compare(site, query, "custom", from: "2022-05-25", to: "2022-05-30") - assert comparison.date_range.first == ~D[2022-05-25] - assert comparison.date_range.last == ~D[2022-05-30] + assert comparison.date_range.first == + DateTime.new!(~D[2022-05-25], ~T[00:00:00], site.timezone) + + assert comparison.date_range.last == + DateTime.new!(~D[2022-05-30], ~T[23:59:59], site.timezone) end test "validates from and to dates" do diff --git a/test/plausible/stats/interval_test.exs b/test/plausible/stats/interval_test.exs index c18771edf..1ec223ed3 100644 --- a/test/plausible/stats/interval_test.exs +++ b/test/plausible/stats/interval_test.exs @@ -2,6 +2,7 @@ defmodule Plausible.Stats.IntervalTest do use Plausible.DataCase, async: true import Plausible.Stats.Interval + alias Plausible.Stats.DateTimeRange test "default_for_period/1" do assert default_for_period("realtime") == "minute" @@ -11,9 +12,13 @@ defmodule Plausible.Stats.IntervalTest do end test "default_for_date_range/1" do - assert default_for_date_range(Date.range(~D[2022-01-01], ~D[2023-01-01])) == "month" - assert default_for_date_range(Date.range(~D[2022-01-01], ~D[2022-01-15])) == "day" - assert default_for_date_range(Date.range(~D[2022-01-01], ~D[2022-01-01])) == "hour" + year = DateTimeRange.new!(~D[2022-01-01], ~D[2023-01-01], "UTC") + fifteen_days = DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-15], "UTC") + day = DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-01], "UTC") + + assert default_for_date_range(year) == "month" + assert default_for_date_range(fifteen_days) == "day" + assert default_for_date_range(day) == "hour" end describe "valid_by_period/1" do diff --git a/test/plausible/stats/query_optimizer_test.exs b/test/plausible/stats/query_optimizer_test.exs index d35b958bc..64d526edd 100644 --- a/test/plausible/stats/query_optimizer_test.exs +++ b/test/plausible/stats/query_optimizer_test.exs @@ -1,7 +1,7 @@ defmodule Plausible.Stats.QueryOptimizerTest do use Plausible.DataCase, async: true - alias Plausible.Stats.{Query, QueryOptimizer} + alias Plausible.Stats.{Query, QueryOptimizer, DateTimeRange} @default_params %{metrics: [:visitors]} @@ -24,7 +24,7 @@ defmodule Plausible.Stats.QueryOptimizerTest do test "adds time and first metric to order_by if order_by not specified" do assert perform(%{ - date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-02-01 00:00:00]), + date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-31], "UTC"), metrics: [:pageviews, :visitors], dimensions: ["time", "event:page"] }).order_by == @@ -35,69 +35,73 @@ defmodule Plausible.Stats.QueryOptimizerTest do describe "update_group_by_time" do test "does nothing if `time` dimension not passed" do assert perform(%{ - date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-05 00:00:00]), + date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-04], "UTC"), dimensions: ["time:month"] }).dimensions == ["time:month"] end test "updating time dimension" do assert perform(%{ - date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]), + date_range: + DateTimeRange.new!( + DateTime.new!(~D[2022-01-01], ~T[00:00:00], "UTC"), + DateTime.new!(~D[2022-01-01], ~T[05:00:00], "UTC") + ), dimensions: ["time"] }).dimensions == ["time:hour"] assert perform(%{ - date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-02 00:00:00]), + date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-01], "UTC"), dimensions: ["time"] }).dimensions == ["time:hour"] assert perform(%{ - date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-02 16:00:00]), + date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-02], "UTC"), dimensions: ["time"] }).dimensions == ["time:hour"] assert perform(%{ - date_range: Date.range(~D[2022-01-01], ~D[2022-01-04]), + date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-03], "UTC"), dimensions: ["time"] }).dimensions == ["time:day"] assert perform(%{ - date_range: Date.range(~D[2022-01-01], ~D[2022-01-10]), + date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-10], "UTC"), dimensions: ["time"] }).dimensions == ["time:day"] assert perform(%{ - date_range: Date.range(~D[2022-01-01], ~D[2022-01-16]), + date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-01-16], "UTC"), dimensions: ["time"] }).dimensions == ["time:day"] assert perform(%{ - date_range: Date.range(~D[2022-01-01], ~D[2022-02-16]), + date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-02-16], "UTC"), dimensions: ["time"] }).dimensions == ["time:week"] assert perform(%{ - date_range: Date.range(~D[2022-01-01], ~D[2022-03-16]), + date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-03-16], "UTC"), dimensions: ["time"] }).dimensions == ["time:week"] assert perform(%{ - date_range: Date.range(~D[2022-01-01], ~D[2022-03-16]), + date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-03-16], "UTC"), dimensions: ["time"] }).dimensions == ["time:week"] assert perform(%{ - date_range: Date.range(~D[2022-01-01], ~D[2023-11-16]), + date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2023-11-16], "UTC"), dimensions: ["time"] }).dimensions == ["time:month"] assert perform(%{ - date_range: Date.range(~D[2022-01-01], ~D[2024-01-16]), + date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2024-01-16], "UTC"), dimensions: ["time"] }).dimensions == ["time:month"] assert perform(%{ - date_range: Date.range(~D[2022-01-01], ~D[2026-01-01]), + date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2026-01-01], "UTC"), dimensions: ["time"] }).dimensions == ["time:month"] end @@ -106,7 +110,11 @@ defmodule Plausible.Stats.QueryOptimizerTest do describe "update_time_in_order_by" do test "updates explicit time dimension in order_by" do assert perform(%{ - date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]), + date_range: + DateTimeRange.new!( + DateTime.new!(~D[2022-01-01], ~T[00:00:00], "UTC"), + DateTime.new!(~D[2022-01-01], ~T[05:00:00], "UTC") + ), dimensions: ["time:hour"], order_by: [{"time", :asc}] }).order_by == [{"time:hour", :asc}] diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exs index 062abe1cd..64775fa19 100644 --- a/test/plausible/stats/query_parser_test.exs +++ b/test/plausible/stats/query_parser_test.exs @@ -1,41 +1,77 @@ defmodule Plausible.Stats.Filters.QueryParserTest do use Plausible.DataCase + alias Plausible.Stats.DateTimeRange alias Plausible.Stats.Filters import Plausible.Stats.Filters.QueryParser setup [:create_user, :create_new_site] - @today ~D[2021-05-05] - @date_range Date.range(@today, @today) + @now DateTime.new!(~D[2021-05-05], ~T[12:30:00], "UTC") + @date_range_realtime %DateTimeRange{ + first: DateTime.new!(~D[2021-05-05], ~T[12:25:00], "UTC"), + last: DateTime.new!(~D[2021-05-05], ~T[12:30:05], "UTC") + } + @date_range_30m %DateTimeRange{ + first: DateTime.new!(~D[2021-05-05], ~T[12:00:00], "UTC"), + last: DateTime.new!(~D[2021-05-05], ~T[12:30:05], "UTC") + } + @date_range_day %DateTimeRange{ + first: DateTime.new!(~D[2021-05-05], ~T[00:00:00], "UTC"), + last: DateTime.new!(~D[2021-05-05], ~T[23:59:59], "UTC") + } + @date_range_7d %DateTimeRange{ + first: DateTime.new!(~D[2021-04-29], ~T[00:00:00], "UTC"), + last: DateTime.new!(~D[2021-05-05], ~T[23:59:59], "UTC") + } + @date_range_30d %DateTimeRange{ + first: DateTime.new!(~D[2021-04-05], ~T[00:00:00], "UTC"), + last: DateTime.new!(~D[2021-05-05], ~T[23:59:59], "UTC") + } + @date_range_month %DateTimeRange{ + first: DateTime.new!(~D[2021-05-01], ~T[00:00:00], "UTC"), + last: DateTime.new!(~D[2021-05-31], ~T[23:59:59], "UTC") + } + @date_range_6mo %DateTimeRange{ + first: DateTime.new!(~D[2020-12-01], ~T[00:00:00], "UTC"), + last: DateTime.new!(~D[2021-05-31], ~T[23:59:59], "UTC") + } + @date_range_year %DateTimeRange{ + first: DateTime.new!(~D[2021-01-01], ~T[00:00:00], "UTC"), + last: DateTime.new!(~D[2021-12-31], ~T[23:59:59], "UTC") + } + @date_range_12mo %DateTimeRange{ + first: DateTime.new!(~D[2020-06-01], ~T[00:00:00], "UTC"), + last: DateTime.new!(~D[2021-05-31], ~T[23:59:59], "UTC") + } - def check_success(params, site, expected_result), - do: check_success(params, site, :public, expected_result) - - def check_success(params, site, schema_type, expected_result) do - assert {:ok, ^expected_result} = parse(site, schema_type, params, @today) + def check_success(params, site, expected_result, schema_type \\ :public) do + assert {:ok, result} = parse(site, schema_type, params, @now) + assert result == expected_result end - def check_error(params, site, expected_error_message), - do: check_error(params, site, :public, expected_error_message) - - def check_error(params, site, schema_type, expected_error_message) do - {:error, message} = parse(site, schema_type, params, @today) + def check_error(params, site, expected_error_message, schema_type \\ :public) do + {:error, message} = parse(site, schema_type, params, @now) assert message == expected_error_message end - def check_date_range(date_range, site, expected_date_range) do - %{"site_id" => site.domain, "metrics" => ["visitors", "events"], "date_range" => date_range} - |> check_success(site, %{ - metrics: [:visitors, :events], - date_range: expected_date_range, - filters: [], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: %{imports: false, time_labels: false}, - preloaded_goals: [] - }) + def check_date_range(date_params, site, expected_date_range, schema_type \\ :public) do + %{"site_id" => site.domain, "metrics" => ["visitors", "events"]} + |> Map.merge(date_params) + |> check_success( + site, + %{ + metrics: [:visitors, :events], + date_range: expected_date_range, + filters: [], + dimensions: [], + order_by: nil, + timezone: site.timezone, + include: %{imports: false, time_labels: false}, + preloaded_goals: [] + }, + schema_type + ) end test "parsing empty map fails", %{site: site} do @@ -48,7 +84,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do %{"site_id" => site.domain, "metrics" => ["visitors", "events"], "date_range" => "all"} |> check_success(site, %{ metrics: [:visitors, :events], - date_range: @date_range, + date_range: @date_range_day, filters: [], dimensions: [], order_by: nil, @@ -77,24 +113,28 @@ defmodule Plausible.Stats.Filters.QueryParserTest do ], "date_range" => "all" } - |> check_success(site, :internal, %{ - metrics: [ - :time_on_page, - :visitors, - :pageviews, - :visits, - :events, - :bounce_rate, - :visit_duration - ], - date_range: @date_range, - filters: [], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: %{imports: false, time_labels: false}, - preloaded_goals: [] - }) + |> check_success( + site, + %{ + metrics: [ + :time_on_page, + :visitors, + :pageviews, + :visits, + :events, + :bounce_rate, + :visit_duration + ], + date_range: @date_range_day, + filters: [], + dimensions: [], + order_by: nil, + timezone: site.timezone, + include: %{imports: false, time_labels: false}, + preloaded_goals: [] + }, + :internal + ) end test "time_on_page is not a valid metric in public API", %{site: site} do @@ -103,7 +143,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do "metrics" => ["time_on_page"], "date_range" => "all" } - |> check_error(site, :public, "#/metrics/0: Invalid metric \"time_on_page\"") + |> check_error(site, "#/metrics/0: Invalid metric \"time_on_page\"") end test "same metric queried multiple times", %{site: site} do @@ -136,18 +176,22 @@ defmodule Plausible.Stats.Filters.QueryParserTest do [Atom.to_string(unquote(operation)), "event:name", ["foo"]] ] } - |> check_success(site, :internal, %{ - metrics: [:visitors], - date_range: @date_range, - filters: [ - [unquote(operation), "event:name", ["foo"]] - ], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: %{imports: false, time_labels: false}, - preloaded_goals: [] - }) + |> check_success( + site, + %{ + metrics: [:visitors], + date_range: @date_range_day, + filters: [ + [unquote(operation), "event:name", ["foo"]] + ], + dimensions: [], + order_by: nil, + timezone: site.timezone, + include: %{imports: false, time_labels: false}, + preloaded_goals: [] + }, + :internal + ) end test "#{operation} filter with invalid clause", %{site: site} do @@ -161,8 +205,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_error( site, - :internal, - "#/filters/0: Invalid filter [\"#{unquote(operation)}\", \"event:name\", \"foo\"]" + "#/filters/0: Invalid filter [\"#{unquote(operation)}\", \"event:name\", \"foo\"]", + :internal ) end end @@ -179,7 +223,6 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_error( site, - :public, "#/filters/0: Invalid filter [\"#{unquote(operation)}\", \"event:name\", [\"foo\"]]" ) end @@ -208,7 +251,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:visitors], - date_range: @date_range, + date_range: @date_range_day, filters: [ [:is, "event:props:foobar", ["value"]] ], @@ -233,7 +276,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:visitors], - date_range: @date_range, + date_range: @date_range_day, filters: [ [:is, "event:#{unquote(dimension)}", ["foo"]] ], @@ -259,7 +302,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:visitors], - date_range: @date_range, + date_range: @date_range_day, filters: [ [:is, "visit:#{unquote(dimension)}", ["ab"]] ], @@ -325,7 +368,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:visitors], - date_range: @date_range, + date_range: @date_range_day, filters: [ [:is, "visit:city", [123, 456]] ], @@ -344,7 +387,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:visitors], - date_range: @date_range, + date_range: @date_range_day, filters: [ [:is, "visit:city", ["123", "456"]] ], @@ -381,7 +424,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:visitors], - date_range: @date_range, + date_range: @date_range_day, filters: [], dimensions: ["time"], order_by: nil, @@ -426,12 +469,12 @@ defmodule Plausible.Stats.Filters.QueryParserTest do ] } - assert {:ok, res} = parse(site, :public, params, @today) + assert {:ok, res} = parse(site, :public, params, @now) expected_timezone = site.timezone assert %{ metrics: [:visitors], - date_range: @date_range, + date_range: @date_range_day, filters: [ [:is, "event:goal", ["Signup", "Visit /thank-you"]] ], @@ -479,41 +522,208 @@ defmodule Plausible.Stats.Filters.QueryParserTest do describe "date range validation" do test "parsing shortcut options", %{site: site} do - check_date_range("day", site, Date.range(~D[2021-05-05], ~D[2021-05-05])) - check_date_range("7d", site, Date.range(~D[2021-04-29], ~D[2021-05-05])) - check_date_range("30d", site, Date.range(~D[2021-04-05], ~D[2021-05-05])) - check_date_range("month", site, Date.range(~D[2021-05-01], ~D[2021-05-31])) - check_date_range("6mo", site, Date.range(~D[2020-12-01], ~D[2021-05-31])) - check_date_range("12mo", site, Date.range(~D[2020-06-01], ~D[2021-05-31])) - check_date_range("year", site, Date.range(~D[2021-01-01], ~D[2021-12-31])) + check_date_range(%{"date_range" => "day"}, site, @date_range_day) + check_date_range(%{"date_range" => "7d"}, site, @date_range_7d) + check_date_range(%{"date_range" => "30d"}, site, @date_range_30d) + check_date_range(%{"date_range" => "month"}, site, @date_range_month) + check_date_range(%{"date_range" => "6mo"}, site, @date_range_6mo) + check_date_range(%{"date_range" => "12mo"}, site, @date_range_12mo) + check_date_range(%{"date_range" => "year"}, site, @date_range_year) + end + + test "30m and realtime are available in internal API", %{site: site} do + check_date_range(%{"date_range" => "30m"}, site, @date_range_30m, :internal) + check_date_range(%{"date_range" => "realtime"}, site, @date_range_realtime, :internal) + end + + test "30m and realtime date_ranges are unavailable in public API", %{ + site: site + } do + for date_range <- ["realtime", "30m"] do + %{"site_id" => site.domain, "metrics" => ["visitors"], "date_range" => date_range} + |> check_error(site, "#/date_range: Invalid date range \"#{date_range}\"") + end end test "parsing `all` with previous data", %{site: site} do site = Map.put(site, :stats_start_date, ~D[2020-01-01]) - check_date_range("all", site, Date.range(~D[2020-01-01], ~D[2021-05-05])) + expected_date_range = DateTimeRange.new!(~D[2020-01-01], ~D[2021-05-05], "UTC") + check_date_range(%{"date_range" => "all"}, site, expected_date_range) end test "parsing `all` with no previous data", %{site: site} do site = Map.put(site, :stats_start_date, nil) - - check_date_range("all", site, Date.range(~D[2021-05-05], ~D[2021-05-05])) + check_date_range(%{"date_range" => "all"}, site, @date_range_day) end - test "parsing custom date range", %{site: site} do + test "parsing custom date range from simple date strings", %{site: site} do + check_date_range(%{"date_range" => ["2021-05-05", "2021-05-05"]}, site, @date_range_day) + end + + test "parsing custom date range from iso8601 timestamps", %{site: site} do check_date_range( - ["2021-05-05", "2021-05-05"], + %{"date_range" => ["2024-01-01T00:00:00 UTC", "2024-01-02T23:59:59 UTC"]}, site, - Date.range(~D[2021-05-05], ~D[2021-05-05]) + DateTimeRange.new!( + DateTime.new!(~D[2024-01-01], ~T[00:00:00], "UTC"), + DateTime.new!(~D[2024-01-02], ~T[23:59:59], "UTC") + ) + ) + + check_date_range( + %{ + "date_range" => [ + "2024-08-29T07:12:34 America/Los_Angeles", + "2024-08-29T10:12:34 America/Los_Angeles" + ] + }, + site, + DateTimeRange.new!( + DateTime.new!(~D[2024-08-29], ~T[07:12:34], "America/Los_Angeles"), + DateTime.new!(~D[2024-08-29], ~T[10:12:34], "America/Los_Angeles") + ) ) end - test "parsing invalid custom date range", %{site: site} do + test "parsing invalid custom date range with invalid dates", %{site: site} do %{"site_id" => site.domain, "date_range" => "foo", "metrics" => ["visitors"]} |> check_error(site, "#/date_range: Invalid date range \"foo\"") %{"site_id" => site.domain, "date_range" => ["21415-00", "eee"], "metrics" => ["visitors"]} |> check_error(site, "#/date_range: Invalid date range [\"21415-00\", \"eee\"]") end + + test "custom date range is invalid when timestamps do not include timezone info", %{ + site: site + } do + %{ + "site_id" => site.domain, + "date_range" => ["2021-02-03T00:00:00", "2021-02-03T23:59:59"], + "metrics" => ["visitors"] + } + |> check_error( + site, + "#/date_range: Invalid date range [\"2021-02-03T00:00:00\", \"2021-02-03T23:59:59\"]" + ) + end + + test "custom date range is invalid when timestamp timezone is invalid", %{site: site} do + %{ + "site_id" => site.domain, + "date_range" => ["2021-02-03T00:00:00 Fake/Timezone", "2021-02-03T23:59:59 UTC"], + "metrics" => ["visitors"] + } + |> check_error( + site, + "Invalid date_range '[\"2021-02-03T00:00:00 Fake/Timezone\", \"2021-02-03T23:59:59 UTC\"]'." + ) + end + + test "custom date range is invalid when date and timestamp are combined", %{site: site} do + %{ + "site_id" => site.domain, + "date_range" => ["2021-02-03T00:00:00 UTC", "2021-02-04"], + "metrics" => ["visitors"] + } + |> check_error( + site, + "Invalid date_range '[\"2021-02-03T00:00:00 UTC\", \"2021-02-04\"]'." + ) + end + + test "custom date range is invalid when timestamp cannot be converted to datetime due to a gap in timezone", + %{site: site} do + %{ + "site_id" => site.domain, + "date_range" => [ + "2024-03-31T03:30:00 Europe/Tallinn", + "2024-04-15T10:00:00 Europe/Tallinn" + ], + "metrics" => ["visitors"] + } + |> check_error( + site, + "Invalid date_range '[\"2024-03-31T03:30:00 Europe/Tallinn\", \"2024-04-15T10:00:00 Europe/Tallinn\"]'." + ) + end + + test "parses date_range relative to date param", %{site: site} do + date = @now |> DateTime.to_date() |> Date.to_string() + + for {date_range_shortcut, expected_date_range} <- [ + {"day", @date_range_day}, + {"7d", @date_range_7d}, + {"30d", @date_range_30d}, + {"month", @date_range_month}, + {"6mo", @date_range_6mo}, + {"12mo", @date_range_12mo}, + {"year", @date_range_year} + ] do + %{"date_range" => date_range_shortcut, "date" => date} + |> check_date_range(site, expected_date_range, :internal) + end + end + + test "date parameter is not available in the public API", %{site: site} do + %{ + "site_id" => site.domain, + "metrics" => ["visitors", "events"], + "date_range" => "month", + "date" => "2021-05-05" + } + |> check_error(site, "#/date: Schema does not allow additional properties.") + end + + test "parses date_range.first into a datetime right after the gap in site.timezone", %{ + site: site + } do + site = %{site | timezone: "America/Santiago"} + + expected_date_range = + DateTimeRange.new!( + DateTime.new!(~D[2022-09-11], ~T[01:00:00], site.timezone), + DateTime.new!(~D[2022-09-11], ~T[23:59:59], site.timezone) + ) + + %{"date_range" => ["2022-09-11", "2022-09-11"]} + |> check_date_range(site, expected_date_range) + end + + test "parses date_range.first into the latest of ambiguous datetimes in site.timezone", %{ + site: site + } do + site = %{site | timezone: "America/Havana"} + + {:ambiguous, _, expected_first_datetime} = + DateTime.new(~D[2023-11-05], ~T[00:00:00], site.timezone) + + expected_date_range = + DateTimeRange.new!( + expected_first_datetime, + DateTime.new!(~D[2023-11-05], ~T[23:59:59], site.timezone) + ) + + %{"date_range" => ["2023-11-05", "2023-11-05"]} + |> check_date_range(site, expected_date_range) + end + + test "parses date_range.last into the earliest of ambiguous datetimes in site.timezone", %{ + site: site + } do + site = %{site | timezone: "America/Asuncion"} + + {:ambiguous, first_dt, _second_dt} = + DateTime.new(~D[2024-03-23], ~T[23:59:59], site.timezone) + + expected_date_range = + DateTimeRange.new!( + DateTime.new!(~D[2024-03-23], ~T[00:00:00], site.timezone), + first_dt + ) + + %{"date_range" => ["2024-03-23", "2024-03-23"]} + |> check_date_range(site, expected_date_range) + end end describe "dimensions validation" do @@ -527,7 +737,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:visitors], - date_range: @date_range, + date_range: @date_range_day, filters: [], dimensions: ["event:#{unquote(dimension)}"], order_by: nil, @@ -548,7 +758,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:visitors], - date_range: @date_range, + date_range: @date_range_day, filters: [], dimensions: ["visit:#{unquote(dimension)}"], order_by: nil, @@ -568,7 +778,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:visitors], - date_range: @date_range, + date_range: @date_range_day, filters: [], dimensions: ["event:props:foobar"], order_by: nil, @@ -629,7 +839,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:visitors, :events], - date_range: @date_range, + date_range: @date_range_day, filters: [], dimensions: [], order_by: [{:events, :desc}, {:visitors, :asc}], @@ -649,7 +859,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:visitors], - date_range: @date_range, + date_range: @date_range_day, filters: [], dimensions: ["event:name"], order_by: [{"event:name", :desc}], @@ -757,7 +967,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do # } # |> check_success(site, %{ # metrics: [:conversion_rate], - # date_range: @date_range, + # date_range: @date_range_day, # filters: [[:is, "event:goal", [event: "Signup"]]], # dimensions: [], # order_by: nil, @@ -777,7 +987,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do # } # |> check_success(site, %{ # metrics: [:conversion_rate], - # date_range: @date_range, + # date_range: @date_range_day, # filters: [], # dimensions: ["event:goal"], # order_by: nil, @@ -799,7 +1009,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do # } # |> check_success(site, %{ # metrics: [:views_per_visit], - # date_range: @date_range, + # date_range: @date_range_day, # filters: [[:is, "event:goal", [event: "Signup"]]], # dimensions: [], # order_by: nil, @@ -846,7 +1056,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:bounce_rate], - date_range: @date_range, + date_range: @date_range_day, filters: [], dimensions: ["visit:device"], order_by: nil, @@ -878,7 +1088,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:bounce_rate], - date_range: @date_range, + date_range: @date_range_day, filters: [], dimensions: ["event:page"], order_by: nil, @@ -897,7 +1107,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do } |> check_success(site, %{ metrics: [:bounce_rate], - date_range: @date_range, + date_range: @date_range_day, filters: [[:is, "event:props:foo", ["(none)"]]], dimensions: [], order_by: nil, diff --git a/test/plausible/stats/query_result_test.exs b/test/plausible/stats/query_result_test.exs index 5e2ce5b4d..f7f3db75c 100644 --- a/test/plausible/stats/query_result_test.exs +++ b/test/plausible/stats/query_result_test.exs @@ -48,8 +48,8 @@ defmodule Plausible.Stats.QueryResultTest do "pageviews" ], "date_range": [ - "2024-01-01", - "2024-02-01" + "2024-01-01T00:00:00 UTC", + "2024-02-01T23:59:59 UTC" ], "filters": [], "dimensions": [], diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exs index ee0517cbd..688c8a14c 100644 --- a/test/plausible/stats/query_test.exs +++ b/test/plausible/stats/query_test.exs @@ -1,6 +1,6 @@ defmodule Plausible.Stats.QueryTest do use Plausible.DataCase, async: true - alias Plausible.Stats.Query + alias Plausible.Stats.{Query, DateTimeRange} setup do user = insert(:user) @@ -19,8 +19,8 @@ defmodule Plausible.Stats.QueryTest do test "keeps current timestamp so that utc_boundaries don't depend on time passing by", %{ site: site } do - q1 = %{now: %NaiveDateTime{}} = Query.from(site, %{"period" => "realtime"}) - q2 = %{now: %NaiveDateTime{}} = Query.from(site, %{"period" => "30m"}) + q1 = %{now: %DateTime{}} = Query.from(site, %{"period" => "realtime"}) + q2 = %{now: %DateTime{}} = Query.from(site, %{"period" => "30m"}) boundaries1 = Plausible.Stats.Time.utc_boundaries(q1, site) boundaries2 = Plausible.Stats.Time.utc_boundaries(q2, site) :timer.sleep(1500) @@ -31,136 +31,190 @@ defmodule Plausible.Stats.QueryTest do test "parses day format", %{site: site} do q = Query.from(site, %{"period" => "day", "date" => "2019-01-01"}) - assert q.date_range.first == ~D[2019-01-01] - assert q.date_range.last == ~D[2019-01-01] + assert q.date_range.first == DateTime.new!(~D[2019-01-01], ~T[00:00:00], site.timezone) + assert q.date_range.last == DateTime.new!(~D[2019-01-01], ~T[23:59:59], site.timezone) assert q.interval == "hour" end test "day format defaults to today", %{site: site} do q = Query.from(site, %{"period" => "day"}) - assert q.date_range.first == Timex.today() - assert q.date_range.last == Timex.today() + expected_first_datetime = Date.utc_today() |> DateTime.new!(~T[00:00:00], site.timezone) + expected_last_datetime = Date.utc_today() |> DateTime.new!(~T[23:59:59], site.timezone) + + assert q.date_range.first == expected_first_datetime + assert q.date_range.last == expected_last_datetime assert q.interval == "hour" end test "parses realtime format", %{site: site} do q = Query.from(site, %{"period" => "realtime"}) - assert q.date_range.first == Timex.today() - assert q.date_range.last == Timex.today() + utc_now = DateTime.shift_zone!(q.now, "Etc/UTC") + + expected_first_datetime = utc_now |> DateTime.shift(minute: -5) + expected_last_datetime = utc_now |> DateTime.shift(second: 5) + + assert q.date_range.first == expected_first_datetime + assert q.date_range.last == expected_last_datetime assert q.period == "realtime" end test "parses month format", %{site: site} do q = Query.from(site, %{"period" => "month", "date" => "2019-01-01"}) - assert q.date_range.first == ~D[2019-01-01] - assert q.date_range.last == ~D[2019-01-31] + assert q.date_range.first == DateTime.new!(~D[2019-01-01], ~T[00:00:00], site.timezone) + assert q.date_range.last == DateTime.new!(~D[2019-01-31], ~T[23:59:59], site.timezone) assert q.interval == "day" end test "parses 6 month format", %{site: site} do q = Query.from(site, %{"period" => "6mo"}) - assert q.date_range.first == - Timex.shift(Timex.today(), months: -5) |> Timex.beginning_of_month() + expected_first_datetime = + q.now + |> DateTime.to_date() + |> Date.shift(month: -5) + |> Date.beginning_of_month() + |> DateTime.new!(~T[00:00:00], site.timezone) - assert q.date_range.last == Timex.today() |> Timex.end_of_month() + expected_last_datetime = + q.now + |> DateTime.to_date() + |> Date.end_of_month() + |> DateTime.new!(~T[23:59:59], site.timezone) + + assert q.date_range.first == expected_first_datetime + assert q.date_range.last == expected_last_datetime assert q.interval == "month" end test "parses 12 month format", %{site: site} do q = Query.from(site, %{"period" => "12mo"}) - assert q.date_range.first == - Timex.shift(Timex.today(), months: -11) |> Timex.beginning_of_month() + expected_first_datetime = + q.now + |> DateTime.to_date() + |> Date.shift(month: -11) + |> Date.beginning_of_month() + |> DateTime.new!(~T[00:00:00], site.timezone) - assert q.date_range.last == Timex.today() |> Timex.end_of_month() + expected_last_datetime = + q.now + |> DateTime.to_date() + |> Date.end_of_month() + |> DateTime.new!(~T[23:59:59], site.timezone) + + assert q.date_range.first == expected_first_datetime + assert q.date_range.last == expected_last_datetime assert q.interval == "month" end test "parses year to date format", %{site: site} do q = Query.from(site, %{"period" => "year"}) - assert q.date_range.first == - Timex.now(site.timezone) |> Timex.to_date() |> Timex.beginning_of_year() + %Date{year: current_year} = DateTime.to_date(q.now) - assert q.date_range.last == - Timex.now(site.timezone) |> Timex.to_date() |> Timex.end_of_year() + expected_first_datetime = + Date.new!(current_year, 1, 1) + |> DateTime.new!(~T[00:00:00], site.timezone) + expected_last_datetime = + Date.new!(current_year, 12, 31) + |> DateTime.new!(~T[23:59:59], site.timezone) + + assert q.date_range.first == expected_first_datetime + assert q.date_range.last == expected_last_datetime assert q.interval == "month" end test "parses all time", %{site: site} do q = Query.from(site, %{"period" => "all"}) - assert q.date_range.first == NaiveDateTime.to_date(site.inserted_at) - assert q.date_range.last == Timex.today() + expected_last_datetime = + q.now + |> DateTime.to_date() + |> DateTime.new!(~T[23:59:59], site.timezone) + + assert DateTime.to_naive(q.date_range.first) == site.inserted_at + assert q.date_range.last == expected_last_datetime assert q.period == "all" assert q.interval == "month" end - test "parses all time in correct timezone", %{site: site} do - site = Map.put(site, :timezone, "America/Cancun") - q = Query.from(site, %{"period" => "all"}) + test "parses all time in site timezone", %{site: site} do + for timezone <- ["Etc/GMT+12", "Etc/GMT-12"] do + site = Map.put(site, :timezone, timezone) + query = Query.from(site, %{"period" => "all"}) - assert q.date_range.first == ~D[2020-01-01] - assert q.date_range.last == Timex.today("America/Cancun") + expected_first_datetime = DateTime.new!(~D[2020-01-01], ~T[00:00:00], site.timezone) + + expected_last_datetime = + Timex.today(site.timezone) + |> DateTime.new!(~T[23:59:59], site.timezone) + + assert query.date_range.first == expected_first_datetime + assert query.date_range.last == expected_last_datetime + end end test "all time shows today if site has no start date", %{site: site} do site = Map.put(site, :stats_start_date, nil) q = Query.from(site, %{"period" => "all"}) - assert q.date_range.first == Timex.today() - assert q.date_range.last == Timex.today() + today = Date.utc_today() + + assert q.date_range == DateTimeRange.new!(today, today, site.timezone) assert q.period == "all" assert q.interval == "hour" end test "all time shows hourly if site is completely new", %{site: site} do - site = Map.put(site, :stats_start_date, Timex.now() |> Timex.to_date()) + site = Map.put(site, :stats_start_date, Date.utc_today()) q = Query.from(site, %{"period" => "all"}) - assert q.date_range.first == Timex.today() - assert q.date_range.last == Timex.today() + today = Date.utc_today() + + assert q.date_range == DateTimeRange.new!(today, today, site.timezone) assert q.period == "all" assert q.interval == "hour" end test "all time shows daily if site is more than a day old", %{site: site} do - site = - Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(days: -1) |> Timex.to_date()) + today = Date.utc_today() + yesterday = today |> Date.shift(day: -1) + + site = Map.put(site, :stats_start_date, yesterday) q = Query.from(site, %{"period" => "all"}) - assert q.date_range.first == Timex.today() |> Timex.shift(days: -1) - assert q.date_range.last == Timex.today() + assert q.date_range == DateTimeRange.new!(yesterday, today, site.timezone) assert q.period == "all" assert q.interval == "day" end test "all time shows monthly if site is more than a month old", %{site: site} do - site = - Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(months: -1) |> Timex.to_date()) + today = Date.utc_today() + last_month = today |> Date.shift(month: -1) + + site = Map.put(site, :stats_start_date, last_month) q = Query.from(site, %{"period" => "all"}) - assert q.date_range.first == Timex.today() |> Timex.shift(months: -1) - assert q.date_range.last == Timex.today() + assert q.date_range == DateTimeRange.new!(last_month, today, site.timezone) assert q.period == "all" assert q.interval == "month" end test "all time uses passed interval different from the default interval", %{site: site} do - site = - Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(months: -1) |> Timex.to_date()) + today = Date.utc_today() + last_month = today |> Date.shift(month: -1) + + site = Map.put(site, :stats_start_date, last_month) q = Query.from(site, %{"period" => "all", "interval" => "week"}) - assert q.date_range.first == Timex.today() |> Timex.shift(months: -1) - assert q.date_range.last == Timex.today() + assert q.date_range == DateTimeRange.new!(last_month, today, site.timezone) assert q.period == "all" assert q.interval == "week" end @@ -172,8 +226,8 @@ defmodule Plausible.Stats.QueryTest do test "parses custom format", %{site: site} do q = Query.from(site, %{"period" => "custom", "from" => "2019-01-01", "to" => "2019-01-15"}) - assert q.date_range.first == ~D[2019-01-01] - assert q.date_range.last == ~D[2019-01-15] + assert q.date_range.first == DateTime.new!(~D[2019-01-01], ~T[00:00:00], site.timezone) + assert q.date_range.last == DateTime.new!(~D[2019-01-15], ~T[23:59:59], site.timezone) assert q.interval == "day" end diff --git a/test/plausible/stats/time_test.exs b/test/plausible/stats/time_test.exs index e368a0cbc..37cb983fb 100644 --- a/test/plausible/stats/time_test.exs +++ b/test/plausible/stats/time_test.exs @@ -2,12 +2,13 @@ defmodule Plausible.Stats.TimeTest do use Plausible.DataCase, async: true import Plausible.Stats.Time + alias Plausible.Stats.DateTimeRange describe "time_labels/1" do test "with time:month dimension" do assert time_labels(%{ dimensions: ["visit:device", "time:month"], - date_range: Date.range(~D[2022-01-17], ~D[2022-02-01]) + date_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-02-01], "UTC") }) == [ "2022-01-01", "2022-02-01" @@ -15,7 +16,7 @@ defmodule Plausible.Stats.TimeTest do assert time_labels(%{ dimensions: ["visit:device", "time:month"], - date_range: Date.range(~D[2022-01-01], ~D[2022-03-07]) + date_range: DateTimeRange.new!(~D[2022-01-01], ~D[2022-03-07], "UTC") }) == [ "2022-01-01", "2022-02-01", @@ -26,7 +27,7 @@ defmodule Plausible.Stats.TimeTest do test "with time:week dimension" do assert time_labels(%{ dimensions: ["time:week"], - date_range: Date.range(~D[2020-12-20], ~D[2021-01-08]) + date_range: DateTimeRange.new!(~D[2020-12-20], ~D[2021-01-08], "UTC") }) == [ "2020-12-20", "2020-12-21", @@ -36,7 +37,7 @@ defmodule Plausible.Stats.TimeTest do assert time_labels(%{ dimensions: ["time:week"], - date_range: Date.range(~D[2020-12-21], ~D[2021-01-03]) + date_range: DateTimeRange.new!(~D[2020-12-21], ~D[2021-01-03], "UTC") }) == [ "2020-12-21", "2020-12-28" @@ -46,7 +47,7 @@ defmodule Plausible.Stats.TimeTest do test "with time:day dimension" do assert time_labels(%{ dimensions: ["time:day"], - date_range: Date.range(~D[2022-01-17], ~D[2022-02-02]) + date_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-02-02], "UTC") }) == [ "2022-01-17", "2022-01-18", @@ -71,7 +72,7 @@ defmodule Plausible.Stats.TimeTest do test "with time:hour dimension" do assert time_labels(%{ dimensions: ["time:hour"], - date_range: Date.range(~D[2022-01-17], ~D[2022-01-17]) + date_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-01-17], "UTC") }) == [ "2022-01-17 00:00:00", "2022-01-17 01:00:00", @@ -101,7 +102,7 @@ defmodule Plausible.Stats.TimeTest do assert time_labels(%{ dimensions: ["time:hour"], - date_range: Date.range(~D[2022-01-17], ~D[2022-01-18]) + date_range: DateTimeRange.new!(~D[2022-01-17], ~D[2022-01-18], "UTC") }) == [ "2022-01-17 00:00:00", "2022-01-17 01:00:00", @@ -154,4 +155,50 @@ defmodule Plausible.Stats.TimeTest do ] end end + + test "with time:minute dimension" do + now = DateTime.new!(~D[2024-01-01], ~T[12:30:57], "UTC") + + # ~U[2024-01-01 12:00:57Z] + first_dt = DateTime.shift(now, minute: -30) + # ~U[2024-01-01 12:31:02Z] + last_dt = DateTime.shift(now, second: 5) + + assert time_labels(%{ + dimensions: ["time:minute"], + now: now, + date_range: DateTimeRange.new!(first_dt, last_dt) + }) == [ + "2024-01-01 12:00:00", + "2024-01-01 12:01:00", + "2024-01-01 12:02:00", + "2024-01-01 12:03:00", + "2024-01-01 12:04:00", + "2024-01-01 12:05:00", + "2024-01-01 12:06:00", + "2024-01-01 12:07:00", + "2024-01-01 12:08:00", + "2024-01-01 12:09:00", + "2024-01-01 12:10:00", + "2024-01-01 12:11:00", + "2024-01-01 12:12:00", + "2024-01-01 12:13:00", + "2024-01-01 12:14:00", + "2024-01-01 12:15:00", + "2024-01-01 12:16:00", + "2024-01-01 12:17:00", + "2024-01-01 12:18:00", + "2024-01-01 12:19:00", + "2024-01-01 12:20:00", + "2024-01-01 12:21:00", + "2024-01-01 12:22:00", + "2024-01-01 12:23:00", + "2024-01-01 12:24:00", + "2024-01-01 12:25:00", + "2024-01-01 12:26:00", + "2024-01-01 12:27:00", + "2024-01-01 12:28:00", + "2024-01-01 12:29:00" + ] + end end diff --git a/test/plausible/timezones_test.exs b/test/plausible/timezones_test.exs index 890d2e2f2..06cae19b8 100644 --- a/test/plausible/timezones_test.exs +++ b/test/plausible/timezones_test.exs @@ -19,15 +19,6 @@ defmodule Plausible.TimezonesTest do refute Enum.empty?(options) end - test "to_utc_datetime/2" do - assert to_utc_datetime(~N[2022-09-11 00:00:00], "Etc/UTC") == ~U[2022-09-11 00:00:00Z] - - assert to_utc_datetime(~N[2022-09-11 00:00:00], "America/Santiago") == - ~U[2022-09-11 00:00:00Z] - - assert to_utc_datetime(~N[2023-10-29 00:00:00], "Atlantic/Azores") == ~U[2023-10-29 01:00:00Z] - end - test "to_date_in_timezone/1" do assert to_date_in_timezone(~D[2021-01-03], "Etc/UTC") == ~D[2021-01-03] assert to_date_in_timezone(~U[2015-01-13 13:00:07Z], "Etc/UTC") == ~D[2015-01-13] diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs index cbfda8e25..6a74ca02a 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs @@ -2896,6 +2896,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do referrer_source: "Google", timestamp: ~N[2021-01-02 00:00:00] ), + build(:pageview, + referrer_source: "Google", + timestamp: ~N[2021-01-02 00:00:00] + ), + build(:pageview, timestamp: ~N[2021-01-03 00:00:00]), build(:pageview, timestamp: ~N[2021-01-03 00:00:00]), build(:pageview, referrer_source: "Twitter", @@ -2912,11 +2917,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do }) assert json_response(conn, 200)["results"] == [ - %{"dimensions" => ["2021-01-01 00:00:00", "Google"], "metrics" => [1]}, - %{"dimensions" => ["2021-01-02 00:00:00", "Google"], "metrics" => [1]}, - %{"dimensions" => ["2021-01-02 00:00:00", "Direct / None"], "metrics" => [1]}, - %{"dimensions" => ["2021-01-03 00:00:00", "Direct / None"], "metrics" => [1]}, - %{"dimensions" => ["2021-01-03 00:00:00", "Twitter"], "metrics" => [1]} + %{"dimensions" => ["2021-01-01", "Google"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-02", "Google"], "metrics" => [2]}, + %{"dimensions" => ["2021-01-02", "Direct / None"], "metrics" => [1]}, + %{"dimensions" => ["2021-01-03", "Direct / None"], "metrics" => [2]}, + %{"dimensions" => ["2021-01-03", "Twitter"], "metrics" => [1]} ] end @@ -2975,4 +2980,32 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do %{"dimensions" => ["United Kingdom", "London", "London"], "metrics" => [1]} ] end + + describe "using the returned query object in a new POST request" do + test "yields the same results for a simple aggregate query", %{conn: conn, site: site} do + Plausible.Site.changeset(site, %{timezone: "Europe/Tallinn"}) + |> Plausible.Repo.update() + + populate_stats(site, [ + build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn1 = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["pageviews"], + "date_range" => "all" + }) + + assert %{"results" => results1, "query" => query} = json_response(conn1, 200) + assert results1 == [%{"metrics" => [3], "dimensions" => []}] + + conn2 = post(conn, "/api/v2/query", query) + + assert %{"results" => results2} = json_response(conn2, 200) + assert results2 == [%{"metrics" => [3], "dimensions" => []}] + end + end end diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs index bfecd730b..c5bfe3555 100644 --- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs @@ -13,8 +13,9 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=realtime&metric=pageviews") - assert %{"plot" => plot} = json_response(conn, 200) + assert %{"plot" => plot, "labels" => labels} = json_response(conn, 200) + assert labels == Enum.to_list(-30..-1) assert Enum.count(plot) == 30 assert Enum.any?(plot, fn pageviews -> pageviews > 0 end) end @@ -1072,6 +1073,28 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do "2021-12-01" => false } end + + test "returns stats for a day with a minute interval", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2023-03-01 12:00:00]) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=day&metric=visitors&date=2023-03-01&interval=minute" + ) + + %{"labels" => labels, "plot" => plot} = json_response(conn, 200) + + assert length(labels) == 24 * 60 + + assert List.first(labels) == "2023-03-01 00:00:00" + assert Enum.at(labels, 1) == "2023-03-01 00:01:00" + assert List.last(labels) == "2023-03-01 23:59:00" + + assert Enum.at(plot, Enum.find_index(labels, &(&1 == "2023-03-01 12:00:00"))) == 1 + end end describe "GET /api/stats/main-graph - comparisons" do