plausible/analytics
1
1
Fork 0
mirror of https://github.com/plausible/analytics.git synced 2024-08-17 05:40:24 +03:00
Code Issues Packages Projects Releases Wiki Activity

APIv2: Aggregates, timeseries, conversion_rate, hostname (#4251)

Browse Source 
* Add some aggregates tests

* Port aggregates tests to do with filtering

* Session metrics can be queried with event: filters

* Solve a typo

* Update a validation message

* Add validations for views_per_visit

* Port an aggregation/imports test

* Optimize time dimension, add tests

* Add first timeseries test, update parsing tests

* Docs for SQL.Expression

* Test timeseries more

* Allow time explicitly in order_by

* Add multiple breakdowns test

* Refactor QueryOptimizer not to care about time dimension placement in dimensions array

* Add test breaking down by event:hostname

* Add hostname filtering logic to QueryOptimizer, unblock some tests

* WIP: Breakdown by goal

* conversion rate logic for query api

* Update more tests

* Set default order_by

* dimension_label

* preloaded_goals in tests

* inline load_goals

* Use Date functions over Timex

* Comments

* is_binary

* Remove special form used in tests

* Fix defmodule

* WIP: Fix memory leak, event:page breakdown logic

* Enable more tests, fix for group_conversion_rate without explicit visitors metric

* Re-enable a partially commented test

* Re-enable a partially commented test

* Get last test passing

* No imports order_by in apiv2

* Add a TODO

* Remove redundant Util call

* Update aggregate.ex

* Remove problematic test
This commit is contained in:
Karl-Aksel Puulmann 2024-06-28 08:59:54 +03:00 committed by GitHub
parent 07a54ef65c
commit 2eeaf7a152
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 2810 additions and 1306 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lib/plausible/stats/aggregate.ex
View File

@ -15,10 +15,7 @@ defmodule Plausible.Stats.Aggregate do
Query.trace(query, metrics)
query_with_metrics = %Plausible.Stats.Query{
query
| metrics: Util.maybe_add_visitors_metric(metrics)
}
query_with_metrics = %Query{query | metrics: metrics}
q = Plausible.Stats.SQL.QueryBuilder.build(query_with_metrics, site)

7
lib/plausible/stats/base.ex
View File

@ -121,6 +121,7 @@ defmodule Plausible.Stats.Base do
defp select_event_metric(:percentage), do: %{}
defp select_event_metric(:conversion_rate), do: %{}
defp select_event_metric(:group_conversion_rate), do: %{}
defp select_event_metric(:total_visitors), do: %{}
defp select_event_metric(unknown), do: raise("Unknown metric: #{unknown}")
@ -344,9 +345,9 @@ defmodule Plausible.Stats.Base do
# only if it's included in the base query - otherwise the total will be based on
# a different data set, making the metric inaccurate. This is why we're using an
# explicit `include_imported` argument here.
defp total_visitors_subquery(site, query, include_imported)
def total_visitors_subquery(site, query, include_imported)
defp total_visitors_subquery(site, query, true = _include_imported) do
def total_visitors_subquery(site, query, true = _include_imported) do
dynamic(
[e],
selected_as(
@ -357,7 +358,7 @@ defmodule Plausible.Stats.Base do
)
end
defp total_visitors_subquery(site, query, false = _include_imported) do
def total_visitors_subquery(site, query, false = _include_imported) do
dynamic([e], selected_as(subquery(total_visitors(site, query)), :__total_visitors))
end

2
lib/plausible/stats/filters/filters.ex
View File

@ -87,6 +87,6 @@ defmodule Plausible.Stats.Filters do
property
|> String.split(":")
|> List.last()
|> String.to_atom()
|> String.to_existing_atom()
end
end

140
lib/plausible/stats/filters/query_parser.ex
View File

@ -5,13 +5,15 @@ defmodule Plausible.Stats.Filters.QueryParser do
alias Plausible.Stats.Filters
alias Plausible.Stats.Query
def parse(site, params) when is_map(params) do
def parse(site, params, now \\ nil) when is_map(params) do
with {:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])),
{:ok, filters} <- parse_filters(Map.get(params, "filters", [])),
{:ok, date_range} <- parse_date_range(site, Map.get(params, "date_range")),
{:ok, date_range} <-
parse_date_range(site, Map.get(params, "date_range"), now || today(site)),
{:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])),
{:ok, order_by} <- parse_order_by(Map.get(params, "order_by")),
{:ok, include} <- parse_include(Map.get(params, "include", %{})),
preloaded_goals <- preload_goals_if_needed(site, filters, dimensions),
query = %{
metrics: metrics,
filters: filters,
@ -19,10 +21,11 @@ defmodule Plausible.Stats.Filters.QueryParser do
dimensions: dimensions,
order_by: order_by,
timezone: site.timezone,
imported_data_requested: Map.get(include, :imports, false)
imported_data_requested: Map.get(include, :imports, false),
preloaded_goals: preloaded_goals
},
:ok <- validate_order_by(query),
:ok <- validate_goal_filters(site, query),
:ok <- validate_goal_filters(query),
:ok <- validate_custom_props_access(site, query),
:ok <- validate_metrics(query) do
{:ok, query}
@ -43,12 +46,14 @@ defmodule Plausible.Stats.Filters.QueryParser do
defp parse_metric("time_on_page"), do: {:ok, :time_on_page}
defp parse_metric("conversion_rate"), do: {:ok, :conversion_rate}
defp parse_metric("group_conversion_rate"), do: {:ok, :group_conversion_rate}
defp parse_metric("visitors"), do: {:ok, :visitors}
defp parse_metric("pageviews"), do: {:ok, :pageviews}
defp parse_metric("events"), do: {:ok, :events}
defp parse_metric("visits"), do: {:ok, :visits}
defp parse_metric("bounce_rate"), do: {:ok, :bounce_rate}
defp parse_metric("visit_duration"), do: {:ok, :visit_duration}
defp parse_metric("views_per_visit"), do: {:ok, :views_per_visit}
defp parse_metric(unknown_metric), do: {:error, "Unknown metric '#{inspect(unknown_metric)}'"}
def parse_filters(filters) when is_list(filters) do
@ -84,7 +89,7 @@ defmodule Plausible.Stats.Filters.QueryParser do
do: parse_clauses_list(filter)
defp parse_clauses_list([_operation, filter_key, list] = filter) when is_list(list) do
all_strings? = Enum.all?(list, &is_bitstring/1)
all_strings? = Enum.all?(list, &is_binary/1)
cond do
filter_key == "event:goal" && all_strings? -> {:ok, [Filters.Utils.wrap_goal_value(list)]}
@ -95,27 +100,62 @@ defmodule Plausible.Stats.Filters.QueryParser do
defp parse_clauses_list(filter), do: {:error, "Invalid filter '#{inspect(filter)}'"}
defp parse_date_range(site, "day") do
today = DateTime.now!(site.timezone) |> DateTime.to_date()
{:ok, Date.range(today, today)}
defp parse_date_range(_site, "day", date) do
{:ok, Date.range(date, date)}
end
defp parse_date_range(_site, "7d"), do: {:ok, "7d"}
defp parse_date_range(_site, "30d"), do: {:ok, "30d"}
defp parse_date_range(_site, "month"), do: {:ok, "month"}
defp parse_date_range(_site, "6mo"), do: {:ok, "6mo"}
defp parse_date_range(_site, "12mo"), do: {:ok, "12mo"}
defp parse_date_range(_site, "year"), do: {:ok, "year"}
defp parse_date_range(_site, "7d", last) do
first = last |> Date.add(-6)
{:ok, Date.range(first, last)}
end
defp parse_date_range(site, "all") do
today = DateTime.now!(site.timezone) |> DateTime.to_date()
defp parse_date_range(_site, "30d", last) do
first = last |> Date.add(-30)
{:ok, Date.range(first, last)}
end
defp parse_date_range(_site, "month", today) do
last = today |> Date.end_of_month()
first = last |> Date.beginning_of_month()
{:ok, Date.range(first, last)}
end
defp parse_date_range(_site, "6mo", today) do
last = today |> Date.end_of_month()
first =
last
|> Timex.shift(months: -5)
|> Date.beginning_of_month()
{:ok, Date.range(first, last)}
end
defp parse_date_range(_site, "12mo", today) do
last = today |> Date.end_of_month()
first =
last
|> Timex.shift(months: -11)
|> Date.beginning_of_month()
{:ok, Date.range(first, last)}
end
defp parse_date_range(_site, "year", today) do
last = today |> Timex.end_of_year()
first = last |> Timex.beginning_of_year()
{:ok, Date.range(first, last)}
end
defp parse_date_range(site, "all", today) do
start_date = Plausible.Sites.stats_start_date(site) || today
{:ok, Date.range(start_date, today)}
end
defp parse_date_range(_site, [from_date_string, to_date_string])
when is_bitstring(from_date_string) and is_bitstring(to_date_string) do
defp parse_date_range(_site, [from_date_string, to_date_string], _date)
when is_binary(from_date_string) and is_binary(to_date_string) do
with {:ok, from_date} <- Date.from_iso8601(from_date_string),
{:ok, to_date} <- Date.from_iso8601(to_date_string) do
{:ok, Date.range(from_date, to_date)}
@ -124,7 +164,10 @@ defmodule Plausible.Stats.Filters.QueryParser do
end
end
defp parse_date_range(_site, unknown), do: {:error, "Invalid date range '#{inspect(unknown)}'"}
defp parse_date_range(_site, unknown, _),
do: {:error, "Invalid date_range '#{inspect(unknown)}'"}
defp today(site), do: DateTime.now!(site.timezone) |> DateTime.to_date()
defp parse_dimensions(dimensions) when is_list(dimensions) do
if length(dimensions) == length(Enum.uniq(dimensions)) do
@ -178,10 +221,9 @@ defmodule Plausible.Stats.Filters.QueryParser do
end
defp parse_time("time"), do: {:ok, "time"}
defp parse_time("time:hour"), do: {:ok, "time:hour"}
defp parse_time("time:day"), do: {:ok, "time:day"}
defp parse_time("time:week"), do: {:ok, "time:week"}
defp parse_time("time:month"), do: {:ok, "time:month"}
defp parse_time("time:year"), do: {:ok, "time:year"}
defp parse_time(_), do: :error
defp parse_order_direction([_, "asc"]), do: {:ok, :asc}
@ -242,7 +284,22 @@ defmodule Plausible.Stats.Filters.QueryParser do
end
end
defp validate_goal_filters(site, query) do
defp preload_goals_if_needed(site, filters, dimensions) do
goal_filters? =
Enum.any?(filters, fn [_, filter_key | _rest] -> filter_key == "event:goal" end)
if goal_filters? or Enum.member?(dimensions, "event:goal") do
Plausible.Goals.for_site(site)
|> Enum.map(fn
%{page_path: path} when is_binary(path) -> {:page, path}
%{event_name: event_name} -> {:event, event_name}
end)
else
[]
end
end
defp validate_goal_filters(query) do
goal_filter_clauses =
Enum.flat_map(query.filters, fn
[_operation, "event:goal", clauses] -> clauses
@ -250,14 +307,7 @@ defmodule Plausible.Stats.Filters.QueryParser do
end)
if length(goal_filter_clauses) > 0 do
configured_goals =
Plausible.Goals.for_site(site)
|> Enum.map(fn
%{page_path: path} when is_binary(path) -> {:page, path}
%{event_name: event_name} -> {:event, event_name}
end)
validate_list(goal_filter_clauses, &validate_goal_filter(&1, configured_goals))
validate_list(goal_filter_clauses, &validate_goal_filter(&1, query.preloaded_goals))
else
:ok
end
@ -298,14 +348,12 @@ defmodule Plausible.Stats.Filters.QueryParser do
end
defp validate_metrics(query) do
validate_list(query.metrics, &validate_metric(&1, query))
with :ok <- validate_list(query.metrics, &validate_metric(&1, query)) do
validate_no_metrics_filters_conflict(query)
end
end
defp validate_metric(:conversion_rate = metric, query) do
defp validate_metric(metric, query) when metric in [:conversion_rate, :group_conversion_rate] do
if Enum.member?(query.dimensions, "event:goal") or
not is_nil(Query.get_filter(query, "event:goal")) do
:ok
@ -314,20 +362,42 @@ defmodule Plausible.Stats.Filters.QueryParser do
end
end
defp validate_metric(:views_per_visit = metric, query) do
cond do
not is_nil(Query.get_filter(query, "event:page")) ->
{:error, "Metric `#{metric}` cannot be queried with a filter on `event:page`"}
length(query.dimensions) > 0 ->
{:error, "Metric `#{metric}` cannot be queried with `dimensions`"}
true ->
:ok
end
end
defp validate_metric(_, _), do: :ok
defp validate_no_metrics_filters_conflict(query) do
{_event_metrics, sessions_metrics, _other_metrics} =
TableDecider.partition_metrics(query.metrics, query)
if Enum.empty?(sessions_metrics) or not TableDecider.event_filters?(query) do
if Enum.empty?(sessions_metrics) or
not event_dimensions_not_allowing_session_metrics?(query.dimensions) do
:ok
else
{:error,
"Session metric(s) `#{sessions_metrics |> Enum.join(", ")}` cannot be queried along with event filters or dimensions"}
"Session metric(s) `#{sessions_metrics |> Enum.join(", ")}` cannot be queried along with event dimensions"}
end
end
def event_dimensions_not_allowing_session_metrics?(dimensions) do
Enum.any?(dimensions, fn
"event:page" -> false
"event:" <> _ -> true
_ -> false
end)
end
defp parse_list(list, parser_function) do
Enum.reduce_while(list, {:ok, []}, fn value, {:ok, results} ->
case parser_function.(value) do

14
lib/plausible/stats/filters/utils.ex
View File

@ -66,4 +66,18 @@ defmodule Plausible.Stats.Filters.Utils do
def unwrap_goal_value(goals) when is_list(goals), do: Enum.map(goals, &unwrap_goal_value/1)
def unwrap_goal_value({:page, page}), do: "Visit " <> page
def unwrap_goal_value({:event, event}), do: event
def split_goals(goals) do
Enum.split_with(goals, fn {type, _} -> type == :event end)
end
def split_goals_query_expressions(goals) do
{event_goals, pageview_goals} = split_goals(goals)
events = Enum.map(event_goals, fn {_, event} -> event end)
page_regexes =
Enum.map(pageview_goals, fn {_, path} -> Plausible.Stats.Base.page_regex(path) end)
{events, page_regexes}
end
end

144
lib/plausible/stats/imported/imported.ex
View File

@ -1,4 +1,5 @@
defmodule Plausible.Stats.Imported do
alias Plausible.Stats.Filters
use Plausible.ClickhouseRepo
import Ecto.Query
@ -266,6 +267,42 @@ defmodule Plausible.Stats.Imported do
def merge_imported(q, _, %Query{include_imported: false}, _), do: q
# Note: Only called for APIv2, old APIs use merge_imported_pageview_goals
def merge_imported(q, site, %Query{dimensions: ["event:goal"]} = query, metrics)
when query.v2 do
{events, page_regexes} = Filters.Utils.split_goals_query_expressions(query.preloaded_goals)
events_q =
"imported_custom_events"
|> Imported.Base.query_imported(site, query)
|> where([i], i.visitors > 0)
|> select_merge([i], %{
dim0: selected_as(fragment("-indexOf(?, ?)", ^events, i.name), :dim0)
})
|> select_imported_metrics(metrics)
|> group_by([], selected_as(:dim0))
|> where([], selected_as(:dim0) != 0)
pages_q =
"imported_pages"
|> Imported.Base.query_imported(site, query)
|> where([i], i.visitors > 0)
|> where(
[i],
fragment("notEmpty(multiMatchAllIndices(?, ?) as indices)", i.page, ^page_regexes)
)
|> join(:array, index in fragment("indices"))
|> group_by([_i, index], index)
|> select_merge([_i, index], %{
dim0: type(fragment("?", index), :integer)
})
|> select_imported_metrics(metrics)
q
|> naive_dimension_join(events_q, metrics)
|> naive_dimension_join(pages_q, metrics)
end
def merge_imported(q, site, %Query{dimensions: [dimension]} = query, metrics)
when dimension in @imported_properties do
dim = Plausible.Stats.Filters.without_prefix(dimension)
@ -289,7 +326,7 @@ defmodule Plausible.Stats.Imported do
dynamic([s, i], s.browser == i.browser and s.browser_version == i.browser_version)
dim ->
dynamic([s, i], field(s, ^dim) == field(i, ^dim))
dynamic([s, i], field(s, ^shortname(query, dim)) == field(i, ^shortname(query, dim)))
end
from(s in Ecto.Query.subquery(q),
@ -299,7 +336,7 @@ defmodule Plausible.Stats.Imported do
)
|> select_joined_dimension(dim, query)
|> select_joined_metrics(metrics)
|> apply_order_by(metrics)
|> apply_order_by(query, metrics)
end
def merge_imported(q, site, %Query{dimensions: []} = query, metrics) do
@ -357,6 +394,10 @@ defmodule Plausible.Stats.Imported do
|> select_merge([i], %{total_visitors: fragment("sum(?)", i.visitors)})
end
# :TRICKY: Handle backwards compatibility with old breakdown module
defp shortname(query, _dim) when query.v2, do: :dim0
defp shortname(_query, dim), do: dim
defp select_imported_metrics(q, []), do: q
defp select_imported_metrics(q, [:visitors | rest]) do
@ -551,63 +592,71 @@ defmodule Plausible.Stats.Imported do
|> select_imported_metrics(rest)
end
defp group_imported_by(q, dim, _query) when dim in [:source, :referrer] do
defp group_imported_by(q, dim, query) when dim in [:source, :referrer] do
q
|> group_by([i], field(i, ^dim))
|> select_merge([i], %{
^dim => fragment("if(empty(?), ?, ?)", field(i, ^dim), @no_ref, field(i, ^dim))
^shortname(query, dim) =>
fragment(
"if(empty(?), ?, ?)",
field(i, ^dim),
@no_ref,
field(i, ^dim)
)
})
end
defp group_imported_by(q, dim, _query)
defp group_imported_by(q, dim, query)
when dim in [:utm_source, :utm_medium, :utm_campaign, :utm_term, :utm_content] do
q
|> group_by([i], field(i, ^dim))
|> where([i], fragment("not empty(?)", field(i, ^dim)))
|> select_merge([i], %{^dim => field(i, ^dim)})
|> select_merge([i], %{^shortname(query, dim) => field(i, ^dim)})
end
defp group_imported_by(q, :page, _query) do
defp group_imported_by(q, :page, query) do
q
|> group_by([i], i.page)
|> select_merge([i], %{page: i.page, time_on_page: sum(i.time_on_page)})
|> select_merge([i], %{^shortname(query, :page) => i.page, time_on_page: sum(i.time_on_page)})
end
defp group_imported_by(q, :country, _query) do
defp group_imported_by(q, :country, query) do
q
|> group_by([i], i.country)
|> where([i], i.country != "ZZ")
|> select_merge([i], %{country: i.country})
|> select_merge([i], %{^shortname(query, :country) => i.country})
end
defp group_imported_by(q, :region, _query) do
defp group_imported_by(q, :region, query) do
q
|> group_by([i], i.region)
|> where([i], i.region != "")
|> select_merge([i], %{region: i.region})
|> select_merge([i], %{^shortname(query, :region) => i.region})
end
defp group_imported_by(q, :city, _query) do
defp group_imported_by(q, :city, query) do
q
|> group_by([i], i.city)
|> where([i], i.city != 0 and not is_nil(i.city))
|> select_merge([i], %{city: i.city})
|> select_merge([i], %{^shortname(query, :city) => i.city})
end
defp group_imported_by(q, dim, _query) when dim in [:device, :browser] do
defp group_imported_by(q, dim, query) when dim in [:device, :browser] do
q
|> group_by([i], field(i, ^dim))
|> select_merge([i], %{
^dim => fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim))
^shortname(query, dim) =>
fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim))
})
end
defp group_imported_by(q, :browser_version, _query) do
defp group_imported_by(q, :browser_version, query) do
q
|> group_by([i], [i.browser, i.browser_version])
|> select_merge([i], %{
browser: fragment("if(empty(?), ?, ?)", i.browser, @not_set, i.browser),
browser_version:
^shortname(query, :browser) =>
fragment("if(empty(?), ?, ?)", i.browser, @not_set, i.browser),
^shortname(query, :browser_version) =>
fragment(
"if(empty(?), ?, ?)",
i.browser_version,
@ -617,20 +666,22 @@ defmodule Plausible.Stats.Imported do
})
end
defp group_imported_by(q, :os, _query) do
defp group_imported_by(q, :os, query) do
q
|> group_by([i], i.operating_system)
|> select_merge([i], %{
os: fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system)
^shortname(query, :os) =>
fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system)
})
end
defp group_imported_by(q, :os_version, _query) do
defp group_imported_by(q, :os_version, query) do
q
|> group_by([i], [i.operating_system, i.operating_system_version])
|> select_merge([i], %{
os: fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system),
os_version:
^shortname(query, :os) =>
fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system),
^shortname(query, :os_version) =>
fragment(
"if(empty(?), ?, ?)",
i.operating_system_version,
@ -640,23 +691,23 @@ defmodule Plausible.Stats.Imported do
})
end
defp group_imported_by(q, dim, _query) when dim in [:entry_page, :exit_page] do
defp group_imported_by(q, dim, query) when dim in [:entry_page, :exit_page] do
q
|> group_by([i], field(i, ^dim))
|> select_merge([i], %{^dim => field(i, ^dim)})
|> select_merge([i], %{^shortname(query, dim) => field(i, ^dim)})
end
defp group_imported_by(q, :name, _query) do
defp group_imported_by(q, :name, query) do
q
|> group_by([i], i.name)
|> select_merge([i], %{name: i.name})
|> select_merge([i], %{^shortname(query, :name) => i.name})
end
defp group_imported_by(q, :url, query) when query.v2 do
q
|> group_by([i], i.link_url)
|> select_merge([i], %{
url: fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)
^shortname(query, :url) => fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)
})
end
@ -672,7 +723,7 @@ defmodule Plausible.Stats.Imported do
q
|> group_by([i], i.path)
|> select_merge([i], %{
path: fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)
^shortname(query, :path) => fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)
})
end
@ -684,9 +735,9 @@ defmodule Plausible.Stats.Imported do
})
end
defp select_joined_dimension(q, :city, _query) do
defp select_joined_dimension(q, :city, query) do
select_merge(q, [s, i], %{
city: fragment("greatest(?,?)", i.city, s.city)
^shortname(query, :city) => fragment("greatest(?,?)", i.city, s.city)
})
end
@ -717,9 +768,15 @@ defmodule Plausible.Stats.Imported do
})
end
defp select_joined_dimension(q, dim, _query) do
defp select_joined_dimension(q, dim, query) do
select_merge(q, [s, i], %{
^dim => fragment("if(empty(?), ?, ?)", field(s, ^dim), field(i, ^dim), field(s, ^dim))
^shortname(query, dim) =>
fragment(
"if(empty(?), ?, ?)",
field(s, ^shortname(query, dim)),
field(i, ^shortname(query, dim)),
field(s, ^shortname(query, dim))
)
})
end
@ -825,10 +882,23 @@ defmodule Plausible.Stats.Imported do
|> select_joined_metrics(rest)
end
defp apply_order_by(q, [:visitors | rest]) do
defp apply_order_by(q, %Query{v2: true}, _), do: q
defp apply_order_by(q, query, [:visitors | rest]) do
order_by(q, [s, i], desc: s.visitors + i.visitors)
|> apply_order_by(rest)
|> apply_order_by(query, rest)
end
defp apply_order_by(q, _), do: q
defp apply_order_by(q, _query, _), do: q
defp naive_dimension_join(q1, q2, metrics) do
from(a in Ecto.Query.subquery(q1),
full_join: b in subquery(q2),
on: a.dim0 == b.dim0,
select: %{
dim0: fragment("coalesce(?, ?)", a.dim0, b.dim0)
}
)
|> select_joined_metrics(metrics)
end
end

1
lib/plausible/stats/metrics.ex
View File

@ -16,6 +16,7 @@ defmodule Plausible.Stats.Metrics do
:visit_duration,
:events,
:conversion_rate,
:group_conversion_rate,
:time_on_page
] ++ on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: [])

17
lib/plausible/stats/query.ex
View File

@ -15,9 +15,10 @@ defmodule Plausible.Stats.Query do
experimental_reduced_joins?: false,
latest_import_end_date: nil,
metrics: [],
order_by: [],
order_by: nil,
timezone: nil,
v2: false
v2: false,
preloaded_goals: []
require OpenTelemetry.Tracer, as: Tracer
alias Plausible.Stats.{Filters, Interval, Imported}
@ -231,6 +232,18 @@ defmodule Plausible.Stats.Query do
|> refresh_imported_opts()
end
def set_metrics(query, metrics) do
query
|> struct!(metrics: metrics)
|> refresh_imported_opts()
end
def set_order_by(query, order_by) do
query
|> struct!(order_by: order_by)
|> refresh_imported_opts()
end
def put_filter(query, filter) do
query
|> struct!(filters: query.filters ++ [filter])

139
lib/plausible/stats/query_optimizer.ex
View File

@ -1,28 +1,155 @@
defmodule Plausible.Stats.QueryOptimizer do
@moduledoc false
@moduledoc """
Methods to manipulate Query for business logic reasons before building an ecto query.
"""
alias Plausible.Stats.Query
alias Plausible.Stats.{Query, TableDecider, Util}
@doc """
This module manipulates an existing query, updating it according to business logic.
For example, it:
1. Figures out what the right granularity to group by time is
2. Adds a missing order_by clause to a query
3. Updating "time" dimension in order_by to the right granularity
"""
def optimize(query) do
Enum.reduce(pipeline(), query, fn step, acc -> step.(acc) end)
end
@doc """
Splits a query into event and sessions subcomponents as not all metrics can be
queried from a single table.
event:page dimension is treated in a special way, doing a breakdown of visit:entry_page
for sessions.
"""
def split(query) do
{event_metrics, sessions_metrics, _other_metrics} =
query.metrics
|> Util.maybe_add_visitors_metric()
|> TableDecider.partition_metrics(query)
{
Query.set_metrics(query, event_metrics),
split_sessions_query(query, sessions_metrics)
}
end
defp pipeline() do
[
&update_group_by_time/1,
&add_missing_order_by/1,
&update_group_by_time/1
&update_time_in_order_by/1,
&extend_hostname_filters_to_visit/1
]
end
defp add_missing_order_by(%Query{order_by: nil} = query) do
%Query{query | order_by: [{hd(query.metrics), :desc}]}
order_by =
case time_dimension(query) do
nil -> [{hd(query.metrics), :desc}]
time_dimension -> [{time_dimension, :asc}, {hd(query.metrics), :desc}]
end
%Query{query | order_by: order_by}
end
defp add_missing_order_by(query), do: query
defp update_group_by_time(%Query{dimensions: ["time" | rest]} = query) do
%Query{query | dimensions: ["time:month" | rest]}
defp update_group_by_time(
%Query{
date_range: %Date.Range{first: first, last: last}
} = query
) do
dimensions =
query.dimensions
|> Enum.map(fn
"time" -> resolve_time_dimension(first, last)
entry -> entry
end)
%Query{query | dimensions: dimensions}
end
defp update_group_by_time(query), do: query
defp resolve_time_dimension(first, last) do
cond do
Timex.diff(last, first, :hours) <= 48 -> "time:hour"
Timex.diff(last, first, :days) <= 40 -> "time:day"
true -> "time:month"
end
end
defp update_time_in_order_by(query) do
order_by =
query.order_by
|> Enum.map(fn
{"time", direction} -> {time_dimension(query), direction}
entry -> entry
end)
%Query{query | order_by: order_by}
end
@dimensions_hostname_map %{
"visit:source" => "visit:entry_page_hostname",
"visit:entry_page" => "visit:entry_page_hostname",
"visit:utm_medium" => "visit:entry_page_hostname",
"visit:utm_source" => "visit:entry_page_hostname",
"visit:utm_campaign" => "visit:entry_page_hostname",
"visit:utm_content" => "visit:entry_page_hostname",
"visit:utm_term" => "visit:entry_page_hostname",
"visit:referrer" => "visit:entry_page_hostname",
"visit:exit_page" => "visit:exit_page_hostname"
}
# To avoid showing referrers across hostnames when event:hostname
# filter is present for breakdowns, add entry/exit page hostname
# filters
defp extend_hostname_filters_to_visit(query) do
hostname_filters =
query.filters
|> Enum.filter(fn [_operation, filter_key | _rest] -> filter_key == "event:hostname" end)
if length(hostname_filters) > 0 do
extra_filters =
query.dimensions
|> Enum.flat_map(&hostname_filters_for_dimension(&1, hostname_filters))
%Query{query | filters: query.filters ++ extra_filters}
else
query
end
end
defp hostname_filters_for_dimension(dimension, hostname_filters) do
if Map.has_key?(@dimensions_hostname_map, dimension) do
filter_key = Map.get(@dimensions_hostname_map, dimension)
hostname_filters
|> Enum.map(fn [operation, _filter_key | rest] -> [operation, filter_key | rest] end)
else
[]
end
end
defp time_dimension(query) do
Enum.find(query.dimensions, &String.starts_with?(&1, "time"))
end
defp split_sessions_query(query, session_metrics) do
dimensions =
query.dimensions
|> Enum.map(fn
"event:page" -> "visit:entry_page"
dimension -> dimension
end)
query
|> Query.set_metrics(session_metrics)
|> Query.set_dimensions(dimensions)
end
end

20
lib/plausible/stats/query_result.ex
View File

@ -1,7 +1,7 @@
defmodule Plausible.Stats.QueryResult do
@moduledoc false
alias Plausible.Stats.SQL.QueryBuilder
alias Plausible.Stats.Util
alias Plausible.Stats.Filters
alias Plausible.Stats.Query
@ -15,7 +15,7 @@ defmodule Plausible.Stats.QueryResult do
results
|> Enum.map(fn entry ->
%{
dimensions: Enum.map(query.dimensions, &Map.get(entry, QueryBuilder.shortname(&1))),
dimensions: Enum.map(query.dimensions, &dimension_label(&1, entry, query)),
metrics: Enum.map(query.metrics, &Map.get(entry, &1))
}
end)
@ -44,6 +44,22 @@ defmodule Plausible.Stats.QueryResult do
defp meta(_), do: %{}
defp dimension_label("event:goal", entry, query) do
{events, paths} = Filters.Utils.split_goals(query.preloaded_goals)
goal_index = Map.get(entry, Util.shortname(query, "event:goal"))
# Closely coupled logic with Plausible.Stats.SQL.Expression.event_goal_join/2
cond do
goal_index < 0 -> Enum.at(events, -goal_index - 1) |> Filters.Utils.unwrap_goal_value()
goal_index > 0 -> Enum.at(paths, goal_index - 1) |> Filters.Utils.unwrap_goal_value()
end
end
defp dimension_label(dimension, entry, query) do
Map.get(entry, Util.shortname(query, dimension))
end
defp serializable_filter([operation, "event:goal", clauses]) do
[operation, "event:goal", Enum.map(clauses, &Filters.Utils.unwrap_goal_value/1)]
end

22
lib/plausible/stats/sql/expression.ex
View File

@ -1,5 +1,8 @@
defmodule Plausible.Stats.SQL.Expression do
@moduledoc false
@moduledoc """
This module is responsible for generating SQL/Ecto expressions
for dimensions used in query select, group_by and order_by.
"""
import Ecto.Query
@ -86,4 +89,21 @@ defmodule Plausible.Stats.SQL.Expression do
def dimension("visit:country", _query), do: dynamic([t], t.country)
def dimension("visit:region", _query), do: dynamic([t], t.region)
def dimension("visit:city", _query), do: dynamic([t], t.city)
defmacro event_goal_join(events, page_regexes) do
quote do
fragment(
"""
arrayPushFront(
CAST(multiMatchAllIndices(?, ?) AS Array(Int64)),
-indexOf(?, ?)
)
""",
e.pathname,
type(^unquote(page_regexes), {:array, :string}),
type(^unquote(events), {:array, :string}),
e.name
)
end
end
end

199
lib/plausible/stats/sql/query_builder.ex
View File

@ -5,47 +5,45 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
import Ecto.Query
import Plausible.Stats.Imported
import Plausible.Stats.Util
alias Plausible.Stats.{Base, Query, TableDecider, Util, Filters, Metrics}
alias Plausible.Stats.{Base, Query, QueryOptimizer, TableDecider, Filters, Metrics}
alias Plausible.Stats.SQL.Expression
require Plausible.Stats.SQL.Expression
def build(query, site) do
{event_metrics, sessions_metrics, _other_metrics} =
query.metrics
|> Util.maybe_add_visitors_metric()
|> TableDecider.partition_metrics(query)
{event_query, sessions_query} = QueryOptimizer.split(query)
event_q = build_events_query(site, event_query)
sessions_q = build_sessions_query(site, sessions_query)
join_query_results(
build_events_query(site, query, event_metrics),
event_metrics,
build_sessions_query(site, query, sessions_metrics),
sessions_metrics,
query
{event_q, event_query},
{sessions_q, sessions_query}
)
end
def shortname(metric) when is_atom(metric), do: metric
def shortname(dimension), do: Plausible.Stats.Filters.without_prefix(dimension)
defp build_events_query(_site, %Query{metrics: []}), do: nil
defp build_events_query(_, _, []), do: nil
defp build_events_query(site, query, event_metrics) do
defp build_events_query(site, events_query) do
q =
from(
e in "events_v2",
where: ^Filters.WhereBuilder.build(:events, site, query),
select: ^Base.select_event_metrics(event_metrics)
where: ^Filters.WhereBuilder.build(:events, site, events_query),
select: ^Base.select_event_metrics(events_query.metrics)
)
on_ee do
q = Plausible.Stats.Sampling.add_query_hint(q, query)
q = Plausible.Stats.Sampling.add_query_hint(q, events_query)
end
q
|> join_sessions_if_needed(site, query)
|> build_group_by(query)
|> merge_imported(site, query, event_metrics)
|> Base.maybe_add_conversion_rate(site, query, event_metrics)
|> join_sessions_if_needed(site, events_query)
|> build_group_by(events_query)
|> merge_imported(site, events_query, events_query.metrics)
|> maybe_add_global_conversion_rate(site, events_query)
|> maybe_add_group_conversion_rate(site, events_query)
end
defp join_sessions_if_needed(q, site, query) do
@ -68,24 +66,24 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
end
end
def build_sessions_query(_, _, []), do: nil
defp build_sessions_query(_site, %Query{metrics: []}), do: nil
def build_sessions_query(site, query, session_metrics) do
defp build_sessions_query(site, sessions_query) do
q =
from(
e in "sessions_v2",
where: ^Filters.WhereBuilder.build(:sessions, site, query),
select: ^Base.select_session_metrics(session_metrics, query)
where: ^Filters.WhereBuilder.build(:sessions, site, sessions_query),
select: ^Base.select_session_metrics(sessions_query.metrics, sessions_query)
)
on_ee do
q = Plausible.Stats.Sampling.add_query_hint(q, query)
q = Plausible.Stats.Sampling.add_query_hint(q, sessions_query)
end
q
|> join_events_if_needed(site, query)
|> build_group_by(query)
|> merge_imported(site, query, session_metrics)
|> join_events_if_needed(site, sessions_query)
|> build_group_by(sessions_query)
|> merge_imported(site, sessions_query, sessions_query.metrics)
end
def join_events_if_needed(q, site, query) do
@ -113,15 +111,30 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
end
defp build_group_by(q, query) do
Enum.reduce(query.dimensions, q, fn dimension, q ->
q
|> select_merge(^%{shortname(dimension) => Expression.dimension(dimension, query)})
|> group_by(^Expression.dimension(dimension, query))
end)
Enum.reduce(query.dimensions, q, &dimension_group_by(&2, query, &1))
end
defp dimension_group_by(q, query, "event:goal" = dimension) do
{events, page_regexes} = Filters.Utils.split_goals_query_expressions(query.preloaded_goals)
from(e in q,
array_join: goal in Expression.event_goal_join(events, page_regexes),
select_merge: %{
^shortname(query, dimension) => fragment("?", goal)
},
group_by: goal,
where: goal != 0
)
end
defp dimension_group_by(q, query, dimension) do
q
|> select_merge(^%{shortname(query, dimension) => Expression.dimension(dimension, query)})
|> group_by(^Expression.dimension(dimension, query))
end
defp build_order_by(q, query, mode) do
Enum.reduce(query.order_by, q, &build_order_by(&2, query, &1, mode))
Enum.reduce(query.order_by || [], q, &build_order_by(&2, query, &1, mode))
end
def build_order_by(q, query, {metric_or_dimension, order_direction}, :inner) do
@ -132,36 +145,36 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
order_direction,
if(
Metrics.metric?(metric_or_dimension),
do: dynamic([], selected_as(^shortname(metric_or_dimension))),
do: dynamic([], selected_as(^shortname(query, metric_or_dimension))),
else: Expression.dimension(metric_or_dimension, query)
)
}
)
end
def build_order_by(q, _query, {metric_or_dimension, order_direction}, :outer) do
def build_order_by(q, query, {metric_or_dimension, order_direction}, :outer) do
order_by(
q,
[t],
^{
order_direction,
dynamic([], selected_as(^shortname(metric_or_dimension)))
dynamic([], selected_as(^shortname(query, metric_or_dimension)))
}
)
end
defmacrop select_join_fields(q, list, table_name) do
defmacrop select_join_fields(q, query, list, table_name) do
quote do
Enum.reduce(unquote(list), unquote(q), fn metric_or_dimension, q ->
select_merge(
q,
^%{
shortname(metric_or_dimension) =>
shortname(unquote(query), metric_or_dimension) =>
dynamic(
[e, s],
selected_as(
field(unquote(table_name), ^shortname(metric_or_dimension)),
^shortname(metric_or_dimension)
field(unquote(table_name), ^shortname(unquote(query), metric_or_dimension)),
^shortname(unquote(query), metric_or_dimension)
)
)
}
@ -170,22 +183,98 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
end
end
defp join_query_results(nil, _, nil, _, _query), do: nil
# Adds conversion_rate metric to query, calculated as
# X / Y where Y is the same breakdown value without goal or props
# filters.
def maybe_add_global_conversion_rate(q, site, query) do
if :conversion_rate in query.metrics do
total_query =
query
|> Query.remove_filters(["event:goal", "event:props"])
|> Query.set_dimensions([])
defp join_query_results(events_q, _, nil, _, query),
do: events_q |> build_order_by(query, :inner)
q
|> select_merge(
^%{
total_visitors: Base.total_visitors_subquery(site, total_query, query.include_imported)
}
)
|> select_merge([e], %{
conversion_rate:
selected_as(
fragment(
"if(? > 0, round(? / ? * 100, 1), 0)",
selected_as(:__total_visitors),
selected_as(:visitors),
selected_as(:__total_visitors)
),
:conversion_rate
)
})
else
q
end
end
defp join_query_results(nil, _, sessions_q, _, query),
do: sessions_q |> build_order_by(query, :inner)
# This function injects a group_conversion_rate metric into
# a dimensional query. It is calculated as X / Y, where:
#
# * X is the number of conversions for a set of dimensions
# result (conversion = number of visitors who
# completed the filtered goal with the filtered
# custom properties).
#
# * Y is the number of all visitors for this set of dimensions
# result without the `event:goal` and `event:props:*`
# filters.
def maybe_add_group_conversion_rate(q, site, query) do
if :group_conversion_rate in query.metrics do
group_totals_query =
query
|> Query.remove_filters(["event:goal", "event:props"])
|> Query.set_metrics([:visitors])
|> Query.set_order_by([])
defp join_query_results(events_q, event_metrics, sessions_q, sessions_metrics, query) do
from(e in subquery(q),
left_join: c in subquery(build(group_totals_query, site)),
on: ^build_group_by_join(query),
select_merge: %{
total_visitors: c.visitors,
group_conversion_rate:
selected_as(
fragment(
"if(? > 0, round(? / ? * 100, 1), 0)",
c.visitors,
e.visitors,
c.visitors
),
:group_conversion_rate
)
}
)
|> select_join_fields(query, query.dimensions, e)
|> select_join_fields(query, List.delete(query.metrics, :group_conversion_rate), e)
else
q
end
end
defp join_query_results({nil, _}, {nil, _}), do: nil
defp join_query_results({events_q, events_query}, {nil, _}),
do: events_q |> build_order_by(events_query, :inner)
defp join_query_results({nil, _}, {sessions_q, sessions_query}),
do: sessions_q |> build_order_by(sessions_query, :inner)
defp join_query_results({events_q, events_query}, {sessions_q, sessions_query}) do
join(subquery(events_q), :left, [e], s in subquery(sessions_q),
on: ^build_group_by_join(query)
on: ^build_group_by_join(events_query)
)
|> select_join_fields(query.dimensions, e)
|> select_join_fields(event_metrics, e)
|> select_join_fields(List.delete(sessions_metrics, :sample_percent), s)
|> build_order_by(query, :outer)
|> select_join_fields(events_query, events_query.dimensions, e)
|> select_join_fields(events_query, events_query.metrics, e)
|> select_join_fields(sessions_query, List.delete(sessions_query.metrics, :sample_percent), s)
|> build_order_by(events_query, :outer)
end
defp build_group_by_join(%Query{dimensions: []}), do: true
@ -193,7 +282,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
defp build_group_by_join(query) do
query.dimensions
|> Enum.map(fn dim ->
dynamic([e, s], field(e, ^shortname(dim)) == field(s, ^shortname(dim)))
dynamic([e, s], field(e, ^shortname(query, dim)) == field(s, ^shortname(query, dim)))
end)
|> Enum.reduce(fn condition, acc -> dynamic([], ^acc and ^condition) end)
end

7
lib/plausible/stats/table_decider.ex
View File

@ -14,12 +14,6 @@ defmodule Plausible.Stats.TableDecider do
|> Enum.any?(&(filters_partitioner(query, &1) == :session))
end
def event_filters?(query) do
query
|> filter_keys()
|> Enum.any?(&(filters_partitioner(query, &1) == :event))
end
def partition_metrics(metrics, query) do
%{
event: event_only_metrics,
@ -64,6 +58,7 @@ defmodule Plausible.Stats.TableDecider do
end
defp metric_partitioner(_, :conversion_rate), do: :event
defp metric_partitioner(_, :group_conversion_rate), do: :event
defp metric_partitioner(_, :average_revenue), do: :event
defp metric_partitioner(_, :total_revenue), do: :event
defp metric_partitioner(_, :pageviews), do: :event

11
lib/plausible/stats/util.ex
View File

@ -41,7 +41,8 @@ defmodule Plausible.Stats.Util do
for any of the other metrics to be calculated.
"""
def maybe_add_visitors_metric(metrics) do
needed? = Enum.any?([:conversion_rate, :time_on_page], &(&1 in metrics))
needed? =
Enum.any?([:conversion_rate, :group_conversion_rate, :time_on_page], &(&1 in metrics))
if needed? and :visitors not in metrics do
metrics ++ [:visitors]
@ -49,4 +50,12 @@ defmodule Plausible.Stats.Util do
metrics
end
end
def shortname(_query, metric) when is_atom(metric), do: metric
def shortname(_query, "time:" <> _), do: :time
def shortname(query, dimension) do
index = Enum.find_index(query.dimensions, &(&1 == dimension))
:"dim#{index}"
end
end

147
test/plausible/stats/query_optimizer_test.exs Normal file
View File

@ -0,0 +1,147 @@
defmodule Plausible.Stats.QueryOptimizerTest do
use Plausible.DataCase, async: true
alias Plausible.Stats.{Query, QueryOptimizer}
@default_params %{metrics: [:visitors]}
def perform(params) do
params = Map.merge(@default_params, params) |> Map.to_list()
struct!(Query, params) |> QueryOptimizer.optimize()
end
describe "add_missing_order_by" do
test "does nothing if order_by passed" do
assert perform(%{order_by: [visitors: :desc]}).order_by == [{:visitors, :desc}]
end
test "adds first metric to order_by if order_by not specified" do
assert perform(%{metrics: [:pageviews, :visitors]}).order_by == [{:pageviews, :desc}]
assert perform(%{metrics: [:pageviews, :visitors], dimensions: ["event:page"]}).order_by ==
[{:pageviews, :desc}]
end
test "adds time and first metric to order_by if order_by not specified" do
assert perform(%{
date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-02-01 00:00:00]),
metrics: [:pageviews, :visitors],
dimensions: ["time", "event:page"]
}).order_by ==
[{"time:day", :asc}, {:pageviews, :desc}]
end
end
describe "update_group_by_time" do
test "does nothing if `time` dimension not passed" do
assert perform(%{
date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-05 00:00:00]),
dimensions: ["time:month"]
}).dimensions == ["time:month"]
end
test "updating time dimension" do
assert perform(%{
date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]),
dimensions: ["time"]
}).dimensions == ["time:hour"]
assert perform(%{
date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-02 00:00:00]),
dimensions: ["time"]
}).dimensions == ["time:hour"]
assert perform(%{
date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-02 16:00:00]),
dimensions: ["time"]
}).dimensions == ["time:hour"]
assert perform(%{
date_range: Date.range(~D[2022-01-01], ~D[2022-01-04]),
dimensions: ["time"]
}).dimensions == ["time:day"]
assert perform(%{
date_range: Date.range(~D[2022-01-01], ~D[2022-01-10]),
dimensions: ["time"]
}).dimensions == ["time:day"]
assert perform(%{
date_range: Date.range(~D[2022-01-01], ~D[2022-01-16]),
dimensions: ["time"]
}).dimensions == ["time:day"]
assert perform(%{
date_range: Date.range(~D[2022-01-01], ~D[2022-02-16]),
dimensions: ["time"]
}).dimensions == ["time:month"]
assert perform(%{
date_range: Date.range(~D[2022-01-01], ~D[2022-03-16]),
dimensions: ["time"]
}).dimensions == ["time:month"]
assert perform(%{
date_range: Date.range(~D[2022-01-01], ~D[2022-03-16]),
dimensions: ["time"]
}).dimensions == ["time:month"]
assert perform(%{
date_range: Date.range(~D[2022-01-01], ~D[2023-11-16]),
dimensions: ["time"]
}).dimensions == ["time:month"]
assert perform(%{
date_range: Date.range(~D[2022-01-01], ~D[2024-01-16]),
dimensions: ["time"]
}).dimensions == ["time:month"]
assert perform(%{
date_range: Date.range(~D[2022-01-01], ~D[2026-01-01]),
dimensions: ["time"]
}).dimensions == ["time:month"]
end
end
describe "update_time_in_order_by" do
test "updates explicit time dimension in order_by" do
assert perform(%{
date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]),
dimensions: ["time:hour"],
order_by: [{"time", :asc}]
}).order_by == [{"time:hour", :asc}]
end
end
describe "extend_hostname_filters_to_visit" do
test "updates filters it filtering by event:hostname and visit:referrer and visit:exit_page dimensions" do
assert perform(%{
date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]),
filters: [
[:is, "event:hostname", ["example.com"]],
[:matches, "event:hostname", ["*.com"]]
],
dimensions: ["visit:referrer", "visit:exit_page"]
}).filters == [
[:is, "event:hostname", ["example.com"]],
[:matches, "event:hostname", ["*.com"]],
[:is, "visit:entry_page_hostname", ["example.com"]],
[:matches, "visit:entry_page_hostname", ["*.com"]],
[:is, "visit:exit_page_hostname", ["example.com"]],
[:matches, "visit:exit_page_hostname", ["*.com"]]
]
end
test "does not update filters if not needed" do
assert perform(%{
date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]),
filters: [
[:is, "event:hostname", ["example.com"]]
],
dimensions: ["time", "event:hostname"]
}).filters == [
[:is, "event:hostname", ["example.com"]]
]
end
end
end

187
test/plausible/stats/query_parser_test.exs
View File

@ -6,17 +6,32 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
setup [:create_user, :create_new_site]
@date_range Date.range(Timex.today(), Timex.today())
@today ~D[2021-05-05]
@date_range Date.range(@today, @today)
def check_success(params, site, expected_result) do
assert parse(site, params) == {:ok, expected_result}
assert parse(site, params, @today) == {:ok, expected_result}
end
def check_error(params, site, expected_error_message) do
{:error, message} = parse(site, params)
{:error, message} = parse(site, params, @today)
assert message =~ expected_error_message
end
def check_date_range(date_range, site, expected_date_range) do
%{"metrics" => ["visitors", "events"], "date_range" => date_range}
|> check_success(site, %{
metrics: [:visitors, :events],
date_range: expected_date_range,
filters: [],
dimensions: [],
order_by: nil,
timezone: site.timezone,
imported_data_requested: false,
preloaded_goals: []
})
end
test "parsing empty map fails", %{site: site} do
%{}
|> check_error(site, "No valid metrics passed")
@ -32,7 +47,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
imported_data_requested: false,
preloaded_goals: []
})
end
@ -69,7 +85,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
imported_data_requested: false,
preloaded_goals: []
})
end
@ -98,7 +115,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
imported_data_requested: false,
preloaded_goals: []
})
end
@ -142,7 +160,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
imported_data_requested: false,
preloaded_goals: []
})
end
@ -165,7 +184,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
imported_data_requested: false,
preloaded_goals: []
})
end
end
@ -189,7 +209,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
imported_data_requested: false,
preloaded_goals: []
})
end
end
@ -240,7 +261,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
imported_data_requested: true
imported_data_requested: true,
preloaded_goals: []
})
end
@ -275,7 +297,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
imported_data_requested: false,
preloaded_goals: [{:page, "/thank-you"}, {:event, "Signup"}]
})
end
@ -303,6 +326,42 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
end
describe "date range validation" do
test "parsing shortcut options", %{site: site} do
check_date_range("day", site, Date.range(~D[2021-05-05], ~D[2021-05-05]))
check_date_range("7d", site, Date.range(~D[2021-04-29], ~D[2021-05-05]))
check_date_range("30d", site, Date.range(~D[2021-04-05], ~D[2021-05-05]))
check_date_range("month", site, Date.range(~D[2021-05-01], ~D[2021-05-31]))
check_date_range("6mo", site, Date.range(~D[2020-12-01], ~D[2021-05-31]))
check_date_range("12mo", site, Date.range(~D[2020-06-01], ~D[2021-05-31]))
check_date_range("year", site, Date.range(~D[2021-01-01], ~D[2021-12-31]))
end
test "parsing `all` with previous data", %{site: site} do
site = Map.put(site, :stats_start_date, ~D[2020-01-01])
check_date_range("all", site, Date.range(~D[2020-01-01], ~D[2021-05-05]))
end
test "parsing `all` with no previous data", %{site: site} do
site = Map.put(site, :stats_start_date, nil)
check_date_range("all", site, Date.range(~D[2021-05-05], ~D[2021-05-05]))
end
test "parsing custom date range", %{site: site} do
check_date_range(
["2021-05-05", "2021-05-05"],
site,
Date.range(~D[2021-05-05], ~D[2021-05-05])
)
end
test "parsing invalid custom date range", %{site: site} do
%{"date_range" => "foo", "metrics" => ["visitors"]}
|> check_error(site, ~r/Invalid date_range '\"foo\"'/)
%{"date_range" => ["21415-00", "eee"], "metrics" => ["visitors"]}
|> check_error(site, ~r/Invalid date_range /)
end
end
describe "dimensions validation" do
@ -320,7 +379,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["event:#{unquote(dimension)}"],
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
imported_data_requested: false,
preloaded_goals: []
})
end
end
@ -339,7 +399,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["visit:#{unquote(dimension)}"],
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
imported_data_requested: false,
preloaded_goals: []
})
end
end
@ -357,7 +418,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["event:props:foobar"],
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
imported_data_requested: false,
preloaded_goals: []
})
end
@ -412,7 +474,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: [{:events, :desc}, {:visitors, :asc}],
timezone: site.timezone,
imported_data_requested: false
imported_data_requested: false,
preloaded_goals: []
})
end
@ -430,7 +493,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["event:name"],
order_by: [{"event:name", :desc}],
timezone: site.timezone,
imported_data_requested: false
imported_data_requested: false,
preloaded_goals: []
})
end
@ -525,7 +589,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: [],
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
imported_data_requested: false,
preloaded_goals: [event: "Signup"]
})
end
@ -544,11 +609,58 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["event:goal"],
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
imported_data_requested: false,
preloaded_goals: [event: "Signup"]
})
end
end
describe "views_per_visit metric" do
test "succeeds with normal filters", %{site: site} do
insert(:goal, %{site: site, event_name: "Signup"})
%{
"metrics" => ["views_per_visit"],
"date_range" => "all",
"filters" => [["is", "event:goal", ["Signup"]]]
}
|> check_success(site, %{
metrics: [:views_per_visit],
date_range: @date_range,
filters: [[:is, "event:goal", [event: "Signup"]]],
dimensions: [],
order_by: nil,
timezone: site.timezone,
imported_data_requested: false,
preloaded_goals: [event: "Signup"]
})
end
test "fails validation if event:page filter specified", %{site: site} do
%{
"metrics" => ["views_per_visit"],
"date_range" => "all",
"filters" => [["is", "event:page", ["/"]]]
}
|> check_error(
site,
~r/Metric `views_per_visit` cannot be queried with a filter on `event:page`/
)
end
test "fails validation with dimensions", %{site: site} do
%{
"metrics" => ["views_per_visit"],
"date_range" => "all",
"dimensions" => ["event:name"]
}
|> check_error(
site,
~r/Metric `views_per_visit` cannot be queried with `dimensions`/
)
end
end
describe "session metrics" do
test "single session metric succeeds", %{site: site} do
%{
@ -563,7 +675,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
dimensions: ["visit:device"],
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
imported_data_requested: false,
preloaded_goals: []
})
end
@ -575,20 +688,44 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
}
|> check_error(
site,
"Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions"
"Session metric(s) `bounce_rate` cannot be queried along with event dimensions"
)
end
test "fails if using session metric with event filter", %{site: site} do
test "does not fail if using session metric with event:page dimension", %{site: site} do
%{
"metrics" => ["bounce_rate"],
"date_range" => "all",
"dimensions" => ["event:page"]
}
|> check_success(site, %{
metrics: [:bounce_rate],
date_range: @date_range,
filters: [],
dimensions: ["event:page"],
order_by: nil,
timezone: site.timezone,
imported_data_requested: false,
preloaded_goals: []
})
end
test "does not fail if using session metric with event filter", %{site: site} do
%{
"metrics" => ["bounce_rate"],
"date_range" => "all",
"filters" => [["is", "event:props:foo", ["(none)"]]]
}
|> check_error(
site,
"Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions"
)
|> check_success(site, %{
metrics: [:bounce_rate],
date_range: @date_range,
filters: [[:is, "event:props:foo", ["(none)"]]],
dimensions: [],
order_by: nil,
timezone: site.timezone,
imported_data_requested: false,
preloaded_goals: []
})
end
end
end

3054
test/plausible_web/controllers/api/external_stats_controller/query_test.exs
View File

File diff suppressed because it is too large Load Diff