From 9c98a3f2e82dbdb5405f4197d21a76195aaba2d2 Mon Sep 17 00:00:00 2001 From: Vini Brasil Date: Mon, 14 Nov 2022 18:41:51 -0300 Subject: [PATCH] Add API support for intervals (#2417) --- lib/plausible/purge.ex | 2 +- lib/plausible/stats/fragments.ex | 62 ++++ lib/plausible/stats/imported.ex | 16 +- lib/plausible/stats/interval.ex | 78 +++++ lib/plausible/stats/query.ex | 24 +- lib/plausible/stats/timeseries.ex | 88 +++++- .../controllers/api/stats_controller.ex | 195 ++++++++++++- test/plausible/imported/imported_test.exs | 46 +++ test/plausible/stats/interval_test.exs | 4 + .../api/stats_controller/main_graph_test.exs | 269 ++++++++++++++++++ .../api/stats_controller/suggestions_test.exs | 5 +- 11 files changed, 748 insertions(+), 41 deletions(-) create mode 100644 lib/plausible/stats/interval.ex create mode 100644 test/plausible/stats/interval_test.exs diff --git a/lib/plausible/purge.ex b/lib/plausible/purge.ex index 2a778a2b0..3bb91c9a0 100644 --- a/lib/plausible/purge.ex +++ b/lib/plausible/purge.ex @@ -5,7 +5,7 @@ defmodule Plausible.Purge do Stats are stored on Clickhouse, and unlike other databases data deletion is done asynchronously. - - [Clickhouse `ALTER TABLE ... DELETE` Statement`](https://clickhouse.com/docs/en/sql-reference/statements/alter/delete) + - [Clickhouse `ALTER TABLE ... DELETE` Statement](https://clickhouse.com/docs/en/sql-reference/statements/alter/delete) - [Synchronicity of `ALTER` Queries](https://clickhouse.com/docs/en/sql-reference/statements/alter/#synchronicity-of-alter-queries) """ diff --git a/lib/plausible/stats/fragments.ex b/lib/plausible/stats/fragments.ex index 692233f10..19808288d 100644 --- a/lib/plausible/stats/fragments.ex +++ b/lib/plausible/stats/fragments.ex @@ -35,6 +35,68 @@ defmodule Plausible.Stats.Fragments do end end + @doc """ + Converts time or date and time to the specified timezone. + + Reference: https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#totimezone + """ + defmacro to_timezone(date, timezone) do + quote do + fragment("toTimeZone(?, ?)", unquote(date), unquote(timezone)) + end + end + + @doc """ + Returns the weekstart for `date`. If the weekstart is before the `not_before` + boundary, `not_before` is returned. + + ## Examples + + In this pseudo-code example, the fragment returns the weekstart. The + `not_before` boundary is set to the past Saturday, which is before the + weekstart, therefore the cap does not apply. + + iex> this_wednesday = ~D[2022-11-09] + ...> past_saturday = ~D[2022-11-05] + ...> weekstart_not_before(this_wednesday, past_saturday) + ~D[2022-11-07] + + + In this other example, the fragment returns Tuesday and not the weekstart. + The `not_before` boundary is set to Tuesday, which is past the weekstart, + therefore the cap applies. + + iex> this_wednesday = ~D[2022-11-09] + ...> this_tuesday = ~D[2022-11-08] + ...> weekstart_not_before(this_wednesday, this_tuesday) + ~D[2022-11-08] + + """ + defmacro weekstart_not_before(date, not_before) do + quote do + fragment( + "if(toMonday(?) < toDate(?), toDate(?), toMonday(?))", + unquote(date), + unquote(not_before), + unquote(not_before), + unquote(date) + ) + end + end + + @doc """ + Same as Plausible.Stats.Fragments.weekstart_not_before/2 but converts dates to + the specified timezone. + """ + defmacro weekstart_not_before(date, not_before, timezone) do + quote do + weekstart_not_before( + to_timezone(unquote(date), unquote(timezone)), + unquote(not_before) + ) + end + end + defmacro __using__(_) do quote do import Plausible.Stats.Fragments diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.ex index 9888bba5f..37d1de675 100644 --- a/lib/plausible/stats/imported.ex +++ b/lib/plausible/stats/imported.ex @@ -2,6 +2,8 @@ defmodule Plausible.Stats.Imported do use Plausible.ClickhouseRepo alias Plausible.Stats.Query import Ecto.Query + import Plausible.Stats.Base + import Plausible.Stats.Fragments @no_ref "Direct / None" @@ -21,7 +23,7 @@ defmodule Plausible.Stats.Imported do select: %{} ) |> select_imported_metrics(metrics) - |> apply_interval(query) + |> apply_interval(query, site.timezone) from(s in Ecto.Query.subquery(native_q), full_join: i in subquery(imported_q), @@ -31,13 +33,21 @@ defmodule Plausible.Stats.Imported do |> select_joined_metrics(metrics) end - defp apply_interval(imported_q, %Plausible.Stats.Query{interval: "month"}) do + defp apply_interval(imported_q, %Plausible.Stats.Query{interval: "month"}, _timezone) do imported_q |> group_by([i], fragment("toStartOfMonth(?)", i.date)) |> select_merge([i], %{date: fragment("toStartOfMonth(?)", i.date)}) end - defp apply_interval(imported_q, _query) do + defp apply_interval(imported_q, %Plausible.Stats.Query{interval: "week"} = query, timezone) do + {first_datetime, _} = utc_boundaries(query, timezone) + + imported_q + |> group_by([i], weekstart_not_before(i.date, ^first_datetime)) + |> select_merge([i], %{date: weekstart_not_before(i.date, ^first_datetime)}) + end + + defp apply_interval(imported_q, _query, _timezone) do imported_q |> group_by([i], i.date) |> select_merge([i], %{date: i.date}) diff --git a/lib/plausible/stats/interval.ex b/lib/plausible/stats/interval.ex new file mode 100644 index 000000000..75f55f78b --- /dev/null +++ b/lib/plausible/stats/interval.ex @@ -0,0 +1,78 @@ +defmodule Plausible.Stats.Interval do + @moduledoc """ + Collection of functions to work with intervals. + + The interval of a query defines the granularity of the data. You can think of + it as a `GROUP BY` clause. Possible values are `minute`, `hour`, `date`, + `week`, and `month`. + """ + + @type t() :: String.t() + @typep period() :: String.t() + + @intervals ~w(minute hour date week month) + @spec list() :: [t()] + def list, do: @intervals + + @spec valid?(term()) :: boolean() + def valid?(interval) do + interval in @intervals + end + + @spec default_for_period(period()) :: t() + @doc """ + Returns the suggested interval for the given time period. + + ## Examples + + iex> Plausible.Stats.Interval.default_for_period("7d") + "date" + + """ + def default_for_period(period) do + case period do + "realtime" -> "minute" + "day" -> "hour" + period when period in ["custom", "7d", "30d", "month"] -> "date" + period when period in ["6mo", "12mo", "year"] -> "month" + end + end + + @allowed_intervals_for_period %{ + "realtime" => ["minute"], + "day" => ["minute", "hour"], + "7d" => ["minute", "hour", "date"], + "month" => ["minute", "hour", "date", "week"], + "30d" => ["minute", "hour", "date", "week"], + "6mo" => ["minute", "hour", "date", "week", "month"], + "12mo" => ["minute", "hour", "date", "week", "month"], + "year" => ["minute", "hour", "date", "week", "month"], + "custom" => ["minute", "hour", "date", "week", "month"], + "all" => ["minute", "hour", "date", "week", "month"] + } + + @spec allowed_for_period?(period(), t()) :: boolean() + @doc """ + Returns whether the given interval is valid for a time period. + + Intervals longer than periods are not supported, e.g. current month stats with + a month interval, or today stats with a week interval. + + ## Examples + + + iex> Plausible.Stats.Interval.allowed_for_period?("month", "date") + true + + iex> Plausible.Stats.Interval.allowed_for_period?("30d", "month") + false + + iex> Plausible.Stats.Interval.allowed_for_period?("realtime", "week") + false + + """ + def allowed_for_period?(period, interval) do + allowed = Map.get(@allowed_intervals_for_period, period, []) + interval in allowed + end +end diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index c645e68af..a53c689e5 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -7,7 +7,7 @@ defmodule Plausible.Stats.Query do include_imported: false @default_sample_threshold 20_000_000 - alias Plausible.Stats.FilterParser + alias Plausible.Stats.{FilterParser, Interval} def shift_back(%__MODULE__{period: "year"} = query, site) do # Querying current year to date @@ -67,7 +67,7 @@ defmodule Plausible.Stats.Query do %__MODULE__{ period: "realtime", - interval: "minute", + interval: params["interval"] || Interval.default_for_period(params["period"]), date_range: Date.range(date, date), filters: FilterParser.parse_filters(params["filters"]), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), @@ -81,7 +81,7 @@ defmodule Plausible.Stats.Query do %__MODULE__{ period: "day", date_range: Date.range(date, date), - interval: "hour", + interval: params["interval"] || Interval.default_for_period(params["period"]), filters: FilterParser.parse_filters(params["filters"]), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) } @@ -95,7 +95,7 @@ defmodule Plausible.Stats.Query do %__MODULE__{ period: "7d", date_range: Date.range(start_date, end_date), - interval: "date", + interval: params["interval"] || Interval.default_for_period(params["period"]), filters: FilterParser.parse_filters(params["filters"]), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) } @@ -109,7 +109,7 @@ defmodule Plausible.Stats.Query do %__MODULE__{ period: "30d", date_range: Date.range(start_date, end_date), - interval: "date", + interval: params["interval"] || Interval.default_for_period(params["period"]), filters: FilterParser.parse_filters(params["filters"]), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) } @@ -125,7 +125,7 @@ defmodule Plausible.Stats.Query do %__MODULE__{ period: "month", date_range: Date.range(start_date, end_date), - interval: "date", + interval: params["interval"] || Interval.default_for_period(params["period"]), filters: FilterParser.parse_filters(params["filters"]), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) } @@ -144,7 +144,7 @@ defmodule Plausible.Stats.Query do %__MODULE__{ period: "6mo", date_range: Date.range(start_date, end_date), - interval: Map.get(params, "interval", "month"), + interval: params["interval"] || Interval.default_for_period(params["period"]), filters: FilterParser.parse_filters(params["filters"]), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) } @@ -163,7 +163,7 @@ defmodule Plausible.Stats.Query do %__MODULE__{ period: "12mo", date_range: Date.range(start_date, end_date), - interval: Map.get(params, "interval", "month"), + interval: params["interval"] || Interval.default_for_period(params["period"]), filters: FilterParser.parse_filters(params["filters"]), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) } @@ -180,7 +180,7 @@ defmodule Plausible.Stats.Query do %__MODULE__{ period: "year", date_range: Date.range(start_date, end_date), - interval: Map.get(params, "interval", "month"), + interval: params["interval"] || Interval.default_for_period(params["period"]), filters: FilterParser.parse_filters(params["filters"]), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) } @@ -199,7 +199,7 @@ defmodule Plausible.Stats.Query do "period" => "custom", "from" => Date.to_iso8601(start_date), "to" => Date.to_iso8601(now), - "interval" => "month" + "interval" => params["interval"] || "month" }) ) |> Map.put(:period, "all") @@ -211,7 +211,7 @@ defmodule Plausible.Stats.Query do "period" => "custom", "from" => Date.to_iso8601(start_date), "to" => Date.to_iso8601(now), - "interval" => "date" + "interval" => params["interval"] || "date" }) ) |> Map.put(:period, "all") @@ -240,7 +240,7 @@ defmodule Plausible.Stats.Query do %__MODULE__{ period: "custom", date_range: Date.range(from_date, to_date), - interval: Map.get(params, "interval", "date"), + interval: params["interval"] || Interval.default_for_period(params["period"]), filters: FilterParser.parse_filters(params["filters"]), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) } diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex index 694a7dd21..dfd94d705 100644 --- a/lib/plausible/stats/timeseries.ex +++ b/lib/plausible/stats/timeseries.ex @@ -50,7 +50,7 @@ defmodule Plausible.Stats.Timeseries do |> ClickhouseRepo.all() end - def buckets(%Query{interval: "month"} = query) do + defp buckets(%Query{interval: "month"} = query) do n_buckets = Timex.diff(query.date_range.last, query.date_range.first, :months) Enum.map(n_buckets..0, fn shift -> @@ -60,23 +60,59 @@ defmodule Plausible.Stats.Timeseries do end) end - def buckets(%Query{interval: "date"} = query) do + defp buckets(%Query{interval: "week"} = query) do + n_buckets = Timex.diff(query.date_range.last, query.date_range.first, :weeks) + + Enum.map(0..n_buckets, fn shift -> + query.date_range.first + |> Timex.shift(weeks: shift) + |> date_or_weekstart(query) + end) + end + + defp buckets(%Query{interval: "date"} = query) do Enum.into(query.date_range, []) end - def buckets(%Query{interval: "hour"} = query) do - Enum.map(0..23, fn step -> - Timex.to_datetime(query.date_range.first) + @full_day_in_hours 23 + defp buckets(%Query{interval: "hour"} = query) do + n_buckets = + if query.date_range.first == query.date_range.last do + @full_day_in_hours + else + Timex.diff(query.date_range.last, query.date_range.first, :hours) + end + + Enum.map(0..n_buckets, fn step -> + query.date_range.first + |> Timex.to_datetime() |> Timex.shift(hours: step) |> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{m}:{s}") end) end - def buckets(%Query{period: "30m", interval: "minute"}) do + defp buckets(%Query{period: "30m", interval: "minute"}) do Enum.into(-30..-1, []) end - def select_bucket(q, site, %Query{interval: "month"}) do + @full_day_in_minutes 1439 + defp buckets(%Query{interval: "minute"} = query) do + n_buckets = + if query.date_range.first == query.date_range.last do + @full_day_in_minutes + else + Timex.diff(query.date_range.last, query.date_range.first, :minutes) + end + + Enum.map(0..n_buckets, fn step -> + query.date_range.first + |> Timex.to_datetime() + |> Timex.shift(minutes: step) + |> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{m}:{s}") + end) + end + + defp select_bucket(q, site, %Query{interval: "month"}) do from( e in q, group_by: fragment("toStartOfMonth(toTimeZone(?, ?))", e.timestamp, ^site.timezone), @@ -87,7 +123,18 @@ defmodule Plausible.Stats.Timeseries do ) end - def select_bucket(q, site, %Query{interval: "date"}) do + defp select_bucket(q, site, %Query{interval: "week"} = query) do + {first_datetime, _} = utc_boundaries(query, site.timezone) + + from( + e in q, + select_merge: %{date: weekstart_not_before(e.timestamp, ^first_datetime, ^site.timezone)}, + group_by: weekstart_not_before(e.timestamp, ^first_datetime, ^site.timezone), + order_by: weekstart_not_before(e.timestamp, ^first_datetime, ^site.timezone) + ) + end + + defp select_bucket(q, site, %Query{interval: "date"}) do from( e in q, group_by: fragment("toDate(toTimeZone(?, ?))", e.timestamp, ^site.timezone), @@ -98,7 +145,7 @@ defmodule Plausible.Stats.Timeseries do ) end - def select_bucket(q, site, %Query{interval: "hour"}) do + defp select_bucket(q, site, %Query{interval: "hour"}) do from( e in q, group_by: fragment("toStartOfHour(toTimeZone(?, ?))", e.timestamp, ^site.timezone), @@ -109,7 +156,7 @@ defmodule Plausible.Stats.Timeseries do ) end - def select_bucket(q, _site, %Query{interval: "minute"}) do + defp select_bucket(q, _site, %Query{interval: "minute", period: "30m"}) do from( e in q, group_by: fragment("dateDiff('minute', now(), ?)", e.timestamp), @@ -120,6 +167,27 @@ defmodule Plausible.Stats.Timeseries do ) end + defp select_bucket(q, site, %Query{interval: "minute"}) do + from( + e in q, + group_by: fragment("toStartOfMinute(toTimeZone(?, ?))", e.timestamp, ^site.timezone), + order_by: fragment("toStartOfMinute(toTimeZone(?, ?))", e.timestamp, ^site.timezone), + select_merge: %{ + date: fragment("toStartOfMinute(toTimeZone(?, ?))", e.timestamp, ^site.timezone) + } + ) + end + + defp date_or_weekstart(date, query) do + weekstart = Timex.beginning_of_week(date) + + if Enum.member?(query.date_range, weekstart) do + weekstart + else + date + end + end + defp empty_row(date, metrics) do Enum.reduce(metrics, %{date: date}, fn metric, row -> case metric do diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 04cf3237d..cfac6aad8 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -5,6 +5,88 @@ defmodule PlausibleWeb.Api.StatsController do alias Plausible.Stats alias Plausible.Stats.{Query, Filters} + @doc """ + Returns a time-series based on given parameters. + + ## Parameters + + This API accepts the following parameters: + + * `period` - x-axis of the graph, e.g. `12mo`, `day`, `custom`. + + * `metric` - y-axis of the graph, e.g. `visits`, `visitors`, `pageviews`. + See the Stats API ["Metrics"](https://plausible.io/docs/stats-api#metrics) + section for more details. Defaults to `visitors`. + + * `interval` - granularity of the time-series data. You can think of it as + a `GROUP BY` clause. Possible values are `minute`, `hour`, `date`, `week`, + and `month`. The default depends on the `period` parameter. Check + `Plausible.Query.from/2` for each default. + + * `filters` - optional filters to drill down data. See the Stats API + ["Filtering"](https://plausible.io/docs/stats-api#filtering) section for + more details. + + * `with_imported` - boolean indicating whether to include Google Analytics + imported data or not. Defaults to `false`. + + Full example: + ```elixir + %{ + "from" => "2021-09-06", + "interval" => "month", + "metric" => "visitors", + "period" => "custom", + "to" => "2021-12-13" + } + ``` + + ## Response + + Returns a map with the following keys: + + * `plot` - list of values for the requested metric representing the y-axis + of the graph. + + * `labels` - list of date times representing the x-axis of the graph. + + * `present_index` - index of the element representing the current date in + `labels` and `plot` lists. + + * `interval` - the interval used for querying. + + * `with_imported` - boolean indicating whether the Google Analytics data + was queried or not. + + * `imported_source` - the source of the imported data, when applicable. + Currently only Google Analytics is supported. + + * `full_intervals` - map of dates indicating whether the interval has been + cut off by the requested date range or not. For example, if looking at a + month week-by-week, some weeks may be cut off by the month boundaries. + It's useful to adjust the graph display slightly in case the interval is + not 'full' so that the user understands why the numbers might be lower for + those partial periods. + + Full example: + ```elixir + %{ + "full_intervals" => %{ + "2021-09-01" => false, + "2021-10-01" => true, + "2021-11-01" => true, + "2021-12-01" => false + }, + "imported_source" => nil, + "interval" => "month", + "labels" => ["2021-09-01", "2021-10-01", "2021-11-01", "2021-12-01"], + "plot" => [0, 0, 0, 0], + "present_index" => nil, + "with_imported" => false + } + ``` + + """ def main_graph(conn, params) do site = conn.assigns[:site] @@ -35,6 +117,7 @@ defmodule PlausibleWeb.Api.StatsController do labels = Enum.map(timeseries_result, fn row -> row[:date] end) present_index = present_index_for(site, query, labels) + full_intervals = build_full_intervals(query, labels) json(conn, %{ plot: plot, @@ -42,14 +125,40 @@ defmodule PlausibleWeb.Api.StatsController do present_index: present_index, interval: query.interval, with_imported: query.include_imported, - imported_source: site.imported_data && site.imported_data.source + imported_source: site.imported_data && site.imported_data.source, + full_intervals: full_intervals }) else - _ -> - bad_request(conn) + {:error, message} when is_binary(message) -> bad_request(conn, message) end end + defp build_full_intervals(%{interval: "week", date_range: range}, labels) do + for label <- labels, into: %{} do + interval_start = Timex.beginning_of_week(label) + interval_end = Timex.end_of_week(label) + + within_interval? = Enum.member?(range, interval_start) && Enum.member?(range, interval_end) + + {label, within_interval?} + end + end + + defp build_full_intervals(%{interval: "month", date_range: range}, labels) do + for label <- labels, into: %{} do + interval_start = Timex.beginning_of_month(label) + interval_end = Timex.end_of_month(label) + + within_interval? = Enum.member?(range, interval_start) && Enum.member?(range, interval_end) + + {label, within_interval?} + end + end + + defp build_full_intervals(_query, _labels) do + nil + end + def top_stats(conn, params) do site = conn.assigns[:site] @@ -66,8 +175,7 @@ defmodule PlausibleWeb.Api.StatsController do imported_source: site.imported_data && site.imported_data.source }) else - _ -> - bad_request(conn) + {:error, message} when is_binary(message) -> bad_request(conn, message) end end @@ -87,6 +195,14 @@ defmodule PlausibleWeb.Api.StatsController do Enum.find_index(dates, &(&1 == current_date)) + "week" -> + current_date = + Timex.now(site.timezone) + |> Timex.to_date() + |> date_or_weekstart(query) + + Enum.find_index(dates, &(&1 == current_date)) + "month" -> current_date = Timex.now(site.timezone) @@ -96,7 +212,21 @@ defmodule PlausibleWeb.Api.StatsController do Enum.find_index(dates, &(&1 == current_date)) "minute" -> - nil + current_date = + Timex.now(site.timezone) + |> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{0m}:00") + + Enum.find_index(dates, &(&1 == current_date)) + end + end + + defp date_or_weekstart(date, query) do + weekstart = Timex.beginning_of_week(date) + + if Enum.member?(query.date_range, weekstart) do + weekstart + else + date end end @@ -935,8 +1065,7 @@ defmodule PlausibleWeb.Api.StatsController do json(conn, Stats.filter_suggestions(site, query, params["filter_name"], params["q"])) else - _ -> - bad_request(conn) + {:error, message} when is_binary(message) -> bad_request(conn, message) end end @@ -1046,19 +1175,57 @@ defmodule PlausibleWeb.Api.StatsController do end end - defp validate_params(%{"date" => date}) do - with {:ok, _} <- Date.from_iso8601(date) do + defp validate_params(params) do + with :ok <- validate_date(params), + :ok <- validate_interval(params), + do: validate_interval_granularity(params) + end + + defp validate_date(params) do + with %{"date" => date} <- params, + {:ok, _} <- Date.from_iso8601(date) do :ok + else + %{} -> + :ok + + {:error, _reason} -> + {:error, + "Failed to parse date argument. Only ISO 8601 dates are allowed, e.g. `2019-09-07`, `2020-01-01`"} end end - defp validate_params(_) do - :ok + defp validate_interval(params) do + with %{"interval" => interval} <- params, + true <- Plausible.Stats.Interval.valid?(interval) do + :ok + else + %{} -> + :ok + + false -> + values = Enum.join(Plausible.Stats.Interval.list(), ", ") + {:error, "Invalid value for interval. Accepted values are: #{values}"} + end end - defp bad_request(conn) do + defp validate_interval_granularity(params) do + with %{"interval" => interval, "period" => period} <- params, + true <- Plausible.Stats.Interval.allowed_for_period?(period, interval) do + :ok + else + %{} -> + :ok + + false -> + {:error, + "Invalid combination of interval and period. Interval must be smaller than the selected period, e.g. `period=day,interval=minute`"} + end + end + + defp bad_request(conn, message) do conn |> put_status(400) - |> json(%{error: "input validation error"}) + |> json(%{error: message}) end end diff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exs index 68700af95..3e58aeb1d 100644 --- a/test/plausible/imported/imported_test.exs +++ b/test/plausible/imported/imported_test.exs @@ -60,6 +60,52 @@ defmodule Plausible.ImportedTest do assert Enum.sum(plot) == 4 end + test "returns data grouped by week", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-31 00:00:00]) + ]) + + import_data( + [ + %{ + dimensions: %{"ga:date" => "20210101"}, + metrics: %{ + "ga:users" => "1", + "ga:pageviews" => "1", + "ga:bounces" => "0", + "ga:sessions" => "1", + "ga:sessionDuration" => "60" + } + }, + %{ + dimensions: %{"ga:date" => "20210131"}, + metrics: %{ + "ga:users" => "1", + "ga:pageviews" => "1", + "ga:bounces" => "0", + "ga:sessions" => "1", + "ga:sessionDuration" => "60" + } + } + ], + site.id, + "imported_visitors" + ) + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true&interval=week" + ) + + assert %{"plot" => plot, "imported_source" => "Google Analytics"} = json_response(conn, 200) + assert Enum.count(plot) == 5 + assert List.first(plot) == 2 + assert List.last(plot) == 2 + assert Enum.sum(plot) == 4 + end + test "Sources are imported", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, diff --git a/test/plausible/stats/interval_test.exs b/test/plausible/stats/interval_test.exs new file mode 100644 index 000000000..635c4efa2 --- /dev/null +++ b/test/plausible/stats/interval_test.exs @@ -0,0 +1,4 @@ +defmodule Plausible.Stats.IntervalTest do + use ExUnit.Case, async: true + doctest Plausible.Stats.Interval +end diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs index 81def8a27..8d80e95af 100644 --- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs @@ -383,4 +383,273 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do assert List.first(plot) == 200 end end + + describe "GET /api/stats/main-graph - varying intervals" do + setup [:create_user, :log_in, :create_new_site] + + test "displays visitors for a month on an hourly scale", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-01 01:01:00]) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=visitors&interval=hour" + ) + + assert %{"plot" => plot} = json_response(conn, 200) + + assert Enum.count(plot) == 721 + assert List.first(plot) == 1 + assert Enum.at(plot, 1) == 1 + end + + test "displays visitors for a day on a minute scale", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:15:01]), + build(:pageview, timestamp: ~N[2021-01-01 00:15:02]) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&metric=visitors&interval=minute" + ) + + assert %{"plot" => plot} = json_response(conn, 200) + + assert Enum.count(plot) == 1440 + assert List.first(plot) == 1 + assert Enum.at(plot, 15) == 2 + end + + test "displays visitors for date range on a minute scale", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:15:01]), + build(:pageview, timestamp: ~N[2021-01-01 00:15:02]), + build(:pageview, timestamp: ~N[2021-01-02 00:10:00]), + build(:pageview, timestamp: ~N[2021-01-02 00:11:01]), + build(:pageview, timestamp: ~N[2021-01-02 01:00:02]), + build(:pageview, timestamp: ~N[2021-01-04 03:10:00]), + build(:pageview, timestamp: ~N[2021-01-04 04:11:01]), + build(:pageview, timestamp: ~N[2021-01-04 05:00:02]) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=custom&from=2021-01-01&to=2021-01-04&metric=visitors&interval=minute" + ) + + assert %{"plot" => plot} = json_response(conn, 200) + + assert Enum.count(plot) == 4321 + assert List.first(plot) == 1 + assert Enum.at(plot, 15) == 2 + assert Enum.at(plot, 1450) == 1 + end + + test "displays visitors for 6mo on a day scale", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-15 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-15 00:00:00]), + build(:pageview, timestamp: ~N[2021-02-15 00:00:00]), + build(:pageview, timestamp: ~N[2021-06-30 01:00:00]) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=6mo&date=2021-06-01&metric=visitors&interval=date" + ) + + assert %{"plot" => plot} = json_response(conn, 200) + + assert Enum.count(plot) == 181 + assert List.first(plot) == 1 + assert Enum.at(plot, 14) == 2 + assert Enum.at(plot, 45) == 1 + assert List.last(plot) == 1 + end + + test "displays visitors for a custom period on a monthly scale", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-15 00:00:00]), + build(:pageview, timestamp: ~N[2021-02-15 00:00:00]), + build(:pageview, timestamp: ~N[2021-06-01 00:00:00]) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=custom&from=2021-01-01&to=2021-06-30&metric=visitors&interval=month" + ) + + assert %{"plot" => plot} = json_response(conn, 200) + + assert Enum.count(plot) == 6 + assert List.first(plot) == 2 + assert Enum.at(plot, 1) == 1 + assert List.last(plot) == 1 + end + + test "returns error when requesting an interval longer than the time period", %{ + conn: conn, + site: site + } do + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&metric=visitors&interval=month" + ) + + assert %{ + "error" => + "Invalid combination of interval and period. Interval must be smaller than the selected period, e.g. `period=day,interval=minute`" + } == json_response(conn, 400) + end + + test "returns error when the interval is not valid", %{ + conn: conn, + site: site + } do + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&metric=visitors&interval=biweekly" + ) + + assert %{ + "error" => + "Invalid value for interval. Accepted values are: minute, hour, date, week, month" + } == json_response(conn, 400) + end + + test "displays visitors for a month on a weekly scale", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:15:01]), + build(:pageview, timestamp: ~N[2021-01-05 00:15:02]) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=visitors&interval=week" + ) + + assert %{"plot" => plot} = json_response(conn, 200) + + assert Enum.count(plot) == 5 + assert List.first(plot) == 2 + assert Enum.at(plot, 1) == 1 + end + + test "shows imperfect week-split month on week scale with full week indicators", %{ + conn: conn, + site: site + } do + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=month&metric=visitors&interval=week&date=2021-09-01" + ) + + assert %{"labels" => labels, "full_intervals" => full_intervals} = json_response(conn, 200) + + assert labels == ["2021-09-01", "2021-09-06", "2021-09-13", "2021-09-20", "2021-09-27"] + + assert full_intervals == %{ + "2021-09-01" => false, + "2021-09-06" => true, + "2021-09-13" => true, + "2021-09-20" => true, + "2021-09-27" => false + } + end + + test "shows half-perfect week-split month on week scale with full week indicators", %{ + conn: conn, + site: site + } do + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=month&metric=visitors&interval=week&date=2021-10-01" + ) + + assert %{"labels" => labels, "full_intervals" => full_intervals} = json_response(conn, 200) + + assert labels == ["2021-10-01", "2021-10-04", "2021-10-11", "2021-10-18", "2021-10-25"] + + assert full_intervals == %{ + "2021-10-01" => false, + "2021-10-04" => true, + "2021-10-11" => true, + "2021-10-18" => true, + "2021-10-25" => true + } + end + + test "shows perfect week-split range on week scale with full week indicators", %{ + conn: conn, + site: site + } do + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=custom&metric=visitors&interval=week&from=2020-12-21&to=2021-02-07" + ) + + assert %{"labels" => labels, "full_intervals" => full_intervals} = json_response(conn, 200) + + assert labels == [ + "2020-12-21", + "2020-12-28", + "2021-01-04", + "2021-01-11", + "2021-01-18", + "2021-01-25", + "2021-02-01" + ] + + assert full_intervals == %{ + "2020-12-21" => true, + "2020-12-28" => true, + "2021-01-04" => true, + "2021-01-11" => true, + "2021-01-18" => true, + "2021-01-25" => true, + "2021-02-01" => true + } + end + + test "shows imperfect month-split period on month scale with full month indicators", %{ + conn: conn, + site: site + } do + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=custom&metric=visitors&interval=month&from=2021-09-06&to=2021-12-13" + ) + + assert %{"labels" => labels, "full_intervals" => full_intervals} = json_response(conn, 200) + + assert labels == ["2021-09-01", "2021-10-01", "2021-11-01", "2021-12-01"] + + assert full_intervals == %{ + "2021-09-01" => false, + "2021-10-01" => true, + "2021-11-01" => true, + "2021-12-01" => false + } + end + end end diff --git a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs index b5513f84b..ff0dbaab0 100644 --- a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs @@ -299,7 +299,10 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do "/api/stats/#{site.domain}/suggestions/prop_value?period=all&date=CLEVER_SECURITY_RESEARCH&filters=#{filters}" ) - assert json_response(conn, 400) == %{"error" => "input validation error"} + assert json_response(conn, 400) == %{ + "error" => + "Failed to parse date argument. Only ISO 8601 dates are allowed, e.g. `2019-09-07`, `2020-01-01`" + } end end end