plausible/analytics
1
1
Fork 0
mirror of https://github.com/plausible/analytics.git synced 2024-08-17 05:40:24 +03:00
Code Issues Packages Projects Releases Wiki Activity

APIv2 - initial PR (#4216)

Browse Source 
* WIP new querying

* WIP: Move some aggregate code under new command

* WIP: Add joins, handling less metrics

* join events table to sessions if needed

* Merge imported results with built query

* Remove dead code

* WIP: /api/v2/query

* Allow grouping by time

* Use JOIN for main query

* Build query result

* update parse_time

* Make joinless order by work

* First test

* more breakdown tests

* Serialize event:goal filters in an json-encodable way/reflection

* Handle inner vs outer ORDER BY clauses properly

* Handle single conversion_rate metric

* Update more tests

* Get parsing tests passing again

* Validate filtered goal filter is configured

* Enable more validation tests

* Enable more event:name breakdown tests

* Enable more breakdown tests

* Validate site has access to custom props

* Validate conversion_rate metric which is only allowed in some situations

* Validate that empty event:props: is not valid

* handle query.dimensions properly in table_decider

* test more validations on metrics/dimensions

* Validate session metrics in combination with event dimension(s)

* Tests cleanup

* Parse include.imports

* Get imports working with new querying

* Make more imports tests work

* Make event:props:path imports-adjacent test work

* Get query imports warning-related tests running

* Remove dead pagination tests

* Solve dead import

* Solve some warnings

* Update aggregate metrics tests

* credo

* Improve test naming

* Lazy goal loading

* Use datetime methods

* Ecto -> SQL module name

* Remove Expression.dimension mode option
This commit is contained in:
Karl-Aksel Puulmann 2024-06-25 09:27:19 +03:00 committed by GitHub
parent 5382020ff0
commit 58a66a952c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 4236 additions and 267 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
extra/lib/plausible/stats/goal/revenue.ex
View File

@ -15,14 +15,20 @@ defmodule Plausible.Stats.Goal.Revenue do
def total_revenue_query() do
dynamic(
[e],
fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)
selected_as(
fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount),
:total_revenue
)
)
end
def average_revenue_query() do
dynamic(
[e],
fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)
selected_as(
fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount),
:average_revenue
)
)
end

15
lib/plausible/stats.ex
View File

@ -1,12 +1,16 @@
defmodule Plausible.Stats do
use Plausible
alias Plausible.Stats.QueryResult
use Plausible.ClickhouseRepo
alias Plausible.Stats.{
Breakdown,
Aggregate,
Timeseries,
CurrentVisitors,
FilterSuggestions
FilterSuggestions,
QueryOptimizer,
SQL
}
use Plausible.DebugReplayInfo
@ -31,6 +35,15 @@ defmodule Plausible.Stats do
CurrentVisitors.current_visitors(site)
end
def query(site, query) do
optimized_query = QueryOptimizer.optimize(query)
optimized_query
|> SQL.QueryBuilder.build(site)
|> ClickhouseRepo.all()
|> QueryResult.from(optimized_query)
end
on_ee do
def funnel(site, query, funnel) do
include_sentry_replay_info()

41
lib/plausible/stats/aggregate.ex
View File

@ -1,7 +1,7 @@
defmodule Plausible.Stats.Aggregate do
use Plausible.ClickhouseRepo
use Plausible
import Plausible.Stats.{Base, Imported}
import Plausible.Stats.Base
import Ecto.Query
alias Plausible.Stats.{Query, Util}
@ -15,23 +15,24 @@ defmodule Plausible.Stats.Aggregate do
Query.trace(query, metrics)
{event_metrics, session_metrics, other_metrics} =
metrics
|> Util.maybe_add_visitors_metric()
|> Plausible.Stats.TableDecider.partition_metrics(query)
query_with_metrics = %Plausible.Stats.Query{
query
| metrics: Util.maybe_add_visitors_metric(metrics)
}
event_task = fn -> aggregate_events(site, query, event_metrics) end
session_task = fn -> aggregate_sessions(site, query, session_metrics) end
q = Plausible.Stats.SQL.QueryBuilder.build(query_with_metrics, site)
time_on_page_task =
if :time_on_page in other_metrics do
if :time_on_page in query_with_metrics.metrics do
fn -> aggregate_time_on_page(site, query) end
else
fn -> %{} end
end
Plausible.ClickhouseRepo.parallel_tasks([session_task, event_task, time_on_page_task])
Plausible.ClickhouseRepo.parallel_tasks([
run_query_task(q),
time_on_page_task
])
|> Enum.reduce(%{}, fn aggregate, task_result -> Map.merge(aggregate, task_result) end)
|> Util.keep_requested_metrics(metrics)
|> cast_revenue_metrics_to_money(currency)
@ -40,24 +41,8 @@ defmodule Plausible.Stats.Aggregate do
|> Enum.into(%{})
end
defp aggregate_events(_, _, []), do: %{}
defp aggregate_events(site, query, metrics) do
from(e in base_event_query(site, query), select: ^select_event_metrics(metrics))
|> merge_imported(site, query, metrics)
|> maybe_add_conversion_rate(site, query, metrics)
|> ClickhouseRepo.one()
end
defp aggregate_sessions(_, _, []), do: %{}
defp aggregate_sessions(site, query, metrics) do
from(e in query_sessions(site, query), select: ^select_session_metrics(metrics, query))
|> filter_converted_sessions(site, query)
|> merge_imported(site, query, metrics)
|> ClickhouseRepo.one()
|> Util.keep_requested_metrics(metrics)
end
defp run_query_task(nil), do: fn -> %{} end
defp run_query_task(q), do: fn -> ClickhouseRepo.one(q) end
defp aggregate_time_on_page(site, query) do
windowed_pages_q =

95
lib/plausible/stats/base.ex
View File

@ -31,7 +31,7 @@ defmodule Plausible.Stats.Base do
end
end
defp query_events(site, query) do
def query_events(site, query) do
q = from(e in "events_v2", where: ^Filters.WhereBuilder.build(:events, site, query))
on_ee do
@ -62,14 +62,21 @@ defmodule Plausible.Stats.Base do
pageviews:
dynamic(
[e],
fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name)
selected_as(
fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name),
:pageviews
)
)
}
end
defp select_event_metric(:events) do
%{
events: dynamic([], fragment("toUInt64(round(count(*) * any(_sample_factor)))"))
events:
dynamic(
[],
selected_as(fragment("toUInt64(round(count(*) * any(_sample_factor)))"), :events)
)
}
end
@ -82,7 +89,13 @@ defmodule Plausible.Stats.Base do
defp select_event_metric(:visits) do
%{
visits:
dynamic([e], fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.session_id))
dynamic(
[e],
selected_as(
fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.session_id),
:visits
)
)
}
end
@ -127,10 +140,13 @@ defmodule Plausible.Stats.Base do
bounce_rate:
dynamic(
[],
fragment(
"toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))",
^condition,
^condition
selected_as(
fragment(
"toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))",
^condition,
^condition
),
:bounce_rate
)
),
__internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))
@ -139,7 +155,14 @@ defmodule Plausible.Stats.Base do
defp select_session_metric(:visits, _query) do
%{
visits: dynamic([s], fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign))
visits:
dynamic(
[s],
selected_as(
fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign),
:visits
)
)
}
end
@ -148,7 +171,10 @@ defmodule Plausible.Stats.Base do
pageviews:
dynamic(
[s],
fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews)
selected_as(
fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews),
:pageviews
)
)
}
end
@ -158,7 +184,10 @@ defmodule Plausible.Stats.Base do
events:
dynamic(
[s],
fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events)
selected_as(
fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events),
:events
)
)
}
end
@ -179,7 +208,13 @@ defmodule Plausible.Stats.Base do
defp select_session_metric(:visit_duration, _query) do
%{
visit_duration:
dynamic([], fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))")),
dynamic(
[],
selected_as(
fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))"),
:visit_duration
)
),
__internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))
}
end
@ -189,7 +224,15 @@ defmodule Plausible.Stats.Base do
views_per_visit:
dynamic(
[s],
fragment("ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)", s.sign, s.pageviews, s.sign)
selected_as(
fragment(
"ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)",
s.sign,
s.pageviews,
s.sign
),
:views_per_visit
)
),
__internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))
}
@ -328,11 +371,14 @@ defmodule Plausible.Stats.Base do
)
|> select_merge(%{
percentage:
fragment(
"if(? > 0, round(? / ? * 100, 1), null)",
selected_as(:__total_visitors),
selected_as(:visitors),
selected_as(:__total_visitors)
selected_as(
fragment(
"if(? > 0, round(? / ? * 100, 1), null)",
selected_as(:__total_visitors),
selected_as(:visitors),
selected_as(:__total_visitors)
),
:percentage
)
})
else
@ -357,11 +403,14 @@ defmodule Plausible.Stats.Base do
)
|> select_merge([e], %{
conversion_rate:
fragment(
"if(? > 0, round(? / ? * 100, 1), 0)",
selected_as(:__total_visitors),
e.visitors,
selected_as(:__total_visitors)
selected_as(
fragment(
"if(? > 0, round(? / ? * 100, 1), 0)",
selected_as(:__total_visitors),
e.visitors,
selected_as(:__total_visitors)
),
:conversion_rate
)
})
else

4
lib/plausible/stats/breakdown.ex
View File

@ -276,8 +276,8 @@ defmodule Plausible.Stats.Breakdown do
defp breakdown_table(%Query{dimensions: ["visit:exit_page"]}, _metrics), do: :session
defp breakdown_table(%Query{dimensions: ["visit:exit_page_hostname"]}, _metrics), do: :session
defp breakdown_table(%Query{dimensions: [dimension]} = query, metrics) do
{_, session_metrics, _} = TableDecider.partition_metrics(metrics, query, dimension)
defp breakdown_table(%Query{dimensions: [_dimension]} = query, metrics) do
{_, session_metrics, _} = TableDecider.partition_metrics(metrics, query)
if not Enum.empty?(session_metrics) do
:session

2
lib/plausible/stats/filters/filters.ex
View File

@ -87,6 +87,6 @@ defmodule Plausible.Stats.Filters do
property
|> String.split(":")
|> List.last()
|> String.to_existing_atom()
|> String.to_atom()
end
end

206
lib/plausible/stats/filters/query_parser.ex
View File

@ -1,22 +1,30 @@
defmodule Plausible.Stats.Filters.QueryParser do
@moduledoc false
alias Plausible.Stats.TableDecider
alias Plausible.Stats.Filters
alias Plausible.Stats.Query
def parse(params) when is_map(params) do
def parse(site, params) when is_map(params) do
with {:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])),
{:ok, filters} <- parse_filters(Map.get(params, "filters", [])),
{:ok, date_range} <- parse_date_range(Map.get(params, "date_range")),
{:ok, date_range} <- parse_date_range(site, Map.get(params, "date_range")),
{:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])),
{:ok, order_by} <- parse_order_by(Map.get(params, "order_by")),
{:ok, include} <- parse_include(Map.get(params, "include", %{})),
query = %{
metrics: metrics,
filters: filters,
date_range: date_range,
dimensions: dimensions,
order_by: order_by
order_by: order_by,
timezone: site.timezone,
imported_data_requested: Map.get(include, :imports, false)
},
:ok <- validate_order_by(query) do
:ok <- validate_order_by(query),
:ok <- validate_goal_filters(site, query),
:ok <- validate_custom_props_access(site, query),
:ok <- validate_metrics(query) do
{:ok, query}
end
end
@ -87,16 +95,26 @@ defmodule Plausible.Stats.Filters.QueryParser do
defp parse_clauses_list(filter), do: {:error, "Invalid filter '#{inspect(filter)}'"}
defp parse_date_range("day"), do: {:ok, "day"}
defp parse_date_range("7d"), do: {:ok, "7d"}
defp parse_date_range("30d"), do: {:ok, "30d"}
defp parse_date_range("month"), do: {:ok, "month"}
defp parse_date_range("6mo"), do: {:ok, "6mo"}
defp parse_date_range("12mo"), do: {:ok, "6mo"}
defp parse_date_range("year"), do: {:ok, "year"}
defp parse_date_range("all"), do: {:ok, "all"}
defp parse_date_range(site, "day") do
today = DateTime.now!(site.timezone) |> DateTime.to_date()
{:ok, Date.range(today, today)}
end
defp parse_date_range([from_date_string, to_date_string])
defp parse_date_range(_site, "7d"), do: {:ok, "7d"}
defp parse_date_range(_site, "30d"), do: {:ok, "30d"}
defp parse_date_range(_site, "month"), do: {:ok, "month"}
defp parse_date_range(_site, "6mo"), do: {:ok, "6mo"}
defp parse_date_range(_site, "12mo"), do: {:ok, "12mo"}
defp parse_date_range(_site, "year"), do: {:ok, "year"}
defp parse_date_range(site, "all") do
today = DateTime.now!(site.timezone) |> DateTime.to_date()
start_date = Plausible.Sites.stats_start_date(site) || today
{:ok, Date.range(start_date, today)}
end
defp parse_date_range(_site, [from_date_string, to_date_string])
when is_bitstring(from_date_string) and is_bitstring(to_date_string) do
with {:ok, from_date} <- Date.from_iso8601(from_date_string),
{:ok, to_date} <- Date.from_iso8601(to_date_string) do
@ -106,13 +124,13 @@ defmodule Plausible.Stats.Filters.QueryParser do
end
end
defp parse_date_range(unknown), do: {:error, "Invalid date range '#{inspect(unknown)}'"}
defp parse_date_range(_site, unknown), do: {:error, "Invalid date range '#{inspect(unknown)}'"}
defp parse_dimensions(dimensions) when is_list(dimensions) do
if length(dimensions) == length(Enum.uniq(dimensions)) do
parse_list(
dimensions,
&parse_filter_key_string(&1, "Invalid dimensions '#{inspect(dimensions)}'")
&parse_dimension_entry(&1, "Invalid dimensions '#{inspect(dimensions)}'")
)
else
{:error, "Some dimensions are listed multiple times"}
@ -121,36 +139,67 @@ defmodule Plausible.Stats.Filters.QueryParser do
defp parse_dimensions(dimensions), do: {:error, "Invalid dimensions '#{inspect(dimensions)}'"}
def parse_order_by(order_by) when is_list(order_by) do
defp parse_order_by(order_by) when is_list(order_by) do
parse_list(order_by, &parse_order_by_entry/1)
end
def parse_order_by(nil), do: {:ok, nil}
def parse_order_by(order_by), do: {:error, "Invalid order_by '#{inspect(order_by)}'"}
defp parse_order_by(nil), do: {:ok, nil}
defp parse_order_by(order_by), do: {:error, "Invalid order_by '#{inspect(order_by)}'"}
def parse_order_by_entry(entry) do
with {:ok, metric_or_dimension} <- parse_metric_or_dimension(entry),
defp parse_order_by_entry(entry) do
with {:ok, value} <- parse_metric_or_dimension(entry),
{:ok, order_direction} <- parse_order_direction(entry) do
{:ok, {metric_or_dimension, order_direction}}
{:ok, {value, order_direction}}
end
end
def parse_metric_or_dimension([metric_or_dimension, _] = entry) do
case {parse_metric(metric_or_dimension), parse_filter_key_string(metric_or_dimension)} do
{{:ok, metric}, _} -> {:ok, metric}
{_, {:ok, dimension}} -> {:ok, dimension}
defp parse_dimension_entry(key, error_message) do
case {
parse_time(key),
parse_filter_key_string(key)
} do
{{:ok, time}, _} -> {:ok, time}
{_, {:ok, filter_key}} -> {:ok, filter_key}
_ -> {:error, error_message}
end
end
defp parse_metric_or_dimension([value, _] = entry) do
case {
parse_time(value),
parse_metric(value),
parse_filter_key_string(value)
} do
{{:ok, time}, _, _} -> {:ok, time}
{_, {:ok, metric}, _} -> {:ok, metric}
{_, _, {:ok, dimension}} -> {:ok, dimension}
_ -> {:error, "Invalid order_by entry '#{inspect(entry)}'"}
end
end
def parse_order_direction([_, "asc"]), do: {:ok, :asc}
def parse_order_direction([_, "desc"]), do: {:ok, :desc}
def parse_order_direction(entry), do: {:error, "Invalid order_by entry '#{inspect(entry)}'"}
defp parse_time("time"), do: {:ok, "time"}
defp parse_time("time:day"), do: {:ok, "time:day"}
defp parse_time("time:week"), do: {:ok, "time:week"}
defp parse_time("time:month"), do: {:ok, "time:month"}
defp parse_time("time:year"), do: {:ok, "time:year"}
defp parse_time(_), do: :error
defp parse_order_direction([_, "asc"]), do: {:ok, :asc}
defp parse_order_direction([_, "desc"]), do: {:ok, :desc}
defp parse_order_direction(entry), do: {:error, "Invalid order_by entry '#{inspect(entry)}'"}
defp parse_include(%{"imports" => value}) when is_boolean(value), do: {:ok, %{imports: value}}
defp parse_include(%{}), do: {:ok, %{}}
defp parse_include(include), do: {:error, "Invalid include passed '#{inspect(include)}'"}
defp parse_filter_key_string(filter_key, error_message \\ "") do
case filter_key do
"event:props:" <> _property_name ->
{:ok, filter_key}
"event:props:" <> property_name ->
if String.length(property_name) > 0 do
{:ok, filter_key}
else
{:error, error_message}
end
"event:" <> key ->
if key in Filters.event_props() do
@ -171,7 +220,7 @@ defmodule Plausible.Stats.Filters.QueryParser do
end
end
def validate_order_by(query) do
defp validate_order_by(query) do
if query.order_by do
valid_values = query.metrics ++ query.dimensions
@ -193,6 +242,92 @@ defmodule Plausible.Stats.Filters.QueryParser do
end
end
defp validate_goal_filters(site, query) do
goal_filter_clauses =
Enum.flat_map(query.filters, fn
[_operation, "event:goal", clauses] -> clauses
_ -> []
end)
if length(goal_filter_clauses) > 0 do
configured_goals =
Plausible.Goals.for_site(site)
|> Enum.map(fn
%{page_path: path} when is_binary(path) -> {:page, path}
%{event_name: event_name} -> {:event, event_name}
end)
validate_list(goal_filter_clauses, &validate_goal_filter(&1, configured_goals))
else
:ok
end
end
defp validate_goal_filter(clause, configured_goals) do
if Enum.member?(configured_goals, clause) do
:ok
else
{:error,
"The goal `#{Filters.Utils.unwrap_goal_value(clause)}` is not configured for this site. Find out how to configure goals here: https://plausible.io/docs/stats-api#filtering-by-goals"}
end
end
defp validate_custom_props_access(site, query) do
allowed_props = Plausible.Props.allowed_for(site, bypass_setup?: true)
validate_custom_props_access(site, query, allowed_props)
end
defp validate_custom_props_access(_site, _query, :all), do: :ok
defp validate_custom_props_access(_site, query, allowed_props) do
valid? =
query.filters
|> Enum.map(fn [_operation, filter_key | _rest] -> filter_key end)
|> Enum.concat(query.dimensions)
|> Enum.all?(fn
"event:props:" <> prop -> prop in allowed_props
_ -> true
end)
if valid? do
:ok
else
{:error, "The owner of this site does not have access to the custom properties feature"}
end
end
defp validate_metrics(query) do
validate_list(query.metrics, &validate_metric(&1, query))
with :ok <- validate_list(query.metrics, &validate_metric(&1, query)) do
validate_no_metrics_filters_conflict(query)
end
end
defp validate_metric(:conversion_rate = metric, query) do
if Enum.member?(query.dimensions, "event:goal") or
not is_nil(Query.get_filter(query, "event:goal")) do
:ok
else
{:error, "Metric `#{metric}` can only be queried with event:goal filters or dimensions"}
end
end
defp validate_metric(_, _), do: :ok
defp validate_no_metrics_filters_conflict(query) do
{_event_metrics, sessions_metrics, _other_metrics} =
TableDecider.partition_metrics(query.metrics, query)
if Enum.empty?(sessions_metrics) or not TableDecider.event_filters?(query) do
:ok
else
{:error,
"Session metric(s) `#{sessions_metrics |> Enum.join(", ")}` cannot be queried along with event filters or dimensions"}
end
end
defp parse_list(list, parser_function) do
Enum.reduce_while(list, {:ok, []}, fn value, {:ok, results} ->
case parser_function.(value) do
@ -201,4 +336,13 @@ defmodule Plausible.Stats.Filters.QueryParser do
end
end)
end
defp validate_list(list, parser_function) do
Enum.reduce_while(list, :ok, fn value, :ok ->
case parser_function.(value) do
:ok -> {:cont, :ok}
{:error, _} = error -> {:halt, error}
end
end)
end
end

72
lib/plausible/stats/imported/imported.ex
View File

@ -274,18 +274,18 @@ defmodule Plausible.Stats.Imported do
site
|> Imported.Base.query_imported(query)
|> where([i], i.visitors > 0)
|> group_imported_by(dim)
|> group_imported_by(dim, query)
|> select_imported_metrics(metrics)
join_on =
case dim do
_ when dim in [:url, :path] ->
_ when dim in [:url, :path] and not query.v2 ->
dynamic([s, i], s.breakdown_prop_value == i.breakdown_prop_value)
:os_version ->
:os_version when not query.v2 ->
dynamic([s, i], s.os == i.os and s.os_version == i.os_version)
:browser_version ->
:browser_version when not query.v2 ->
dynamic([s, i], s.browser == i.browser and s.browser_version == i.browser_version)
dim ->
@ -297,7 +297,7 @@ defmodule Plausible.Stats.Imported do
on: ^join_on,
select: %{}
)
|> select_joined_dimension(dim)
|> select_joined_dimension(dim, query)
|> select_joined_metrics(metrics)
|> apply_order_by(metrics)
end
@ -344,7 +344,7 @@ defmodule Plausible.Stats.Imported do
on: s.name == i.name,
select: %{}
)
|> select_joined_dimension(:name)
|> select_joined_dimension(:name, query)
|> select_joined_metrics(metrics)
else
q
@ -551,7 +551,7 @@ defmodule Plausible.Stats.Imported do
|> select_imported_metrics(rest)
end
defp group_imported_by(q, dim) when dim in [:source, :referrer] do
defp group_imported_by(q, dim, _query) when dim in [:source, :referrer] do
q
|> group_by([i], field(i, ^dim))
|> select_merge([i], %{
@ -559,7 +559,7 @@ defmodule Plausible.Stats.Imported do
})
end
defp group_imported_by(q, dim)
defp group_imported_by(q, dim, _query)
when dim in [:utm_source, :utm_medium, :utm_campaign, :utm_term, :utm_content] do
q
|> group_by([i], field(i, ^dim))
@ -567,34 +567,34 @@ defmodule Plausible.Stats.Imported do
|> select_merge([i], %{^dim => field(i, ^dim)})
end
defp group_imported_by(q, :page) do
defp group_imported_by(q, :page, _query) do
q
|> group_by([i], i.page)
|> select_merge([i], %{page: i.page, time_on_page: sum(i.time_on_page)})
end
defp group_imported_by(q, :country) do
defp group_imported_by(q, :country, _query) do
q
|> group_by([i], i.country)
|> where([i], i.country != "ZZ")
|> select_merge([i], %{country: i.country})
end
defp group_imported_by(q, :region) do
defp group_imported_by(q, :region, _query) do
q
|> group_by([i], i.region)
|> where([i], i.region != "")
|> select_merge([i], %{region: i.region})
end
defp group_imported_by(q, :city) do
defp group_imported_by(q, :city, _query) do
q
|> group_by([i], i.city)
|> where([i], i.city != 0 and not is_nil(i.city))
|> select_merge([i], %{city: i.city})
end
defp group_imported_by(q, dim) when dim in [:device, :browser] do
defp group_imported_by(q, dim, _query) when dim in [:device, :browser] do
q
|> group_by([i], field(i, ^dim))
|> select_merge([i], %{
@ -602,7 +602,7 @@ defmodule Plausible.Stats.Imported do
})
end
defp group_imported_by(q, :browser_version) do
defp group_imported_by(q, :browser_version, _query) do
q
|> group_by([i], [i.browser, i.browser_version])
|> select_merge([i], %{
@ -617,7 +617,7 @@ defmodule Plausible.Stats.Imported do
})
end
defp group_imported_by(q, :os) do
defp group_imported_by(q, :os, _query) do
q
|> group_by([i], i.operating_system)
|> select_merge([i], %{
@ -625,7 +625,7 @@ defmodule Plausible.Stats.Imported do
})
end
defp group_imported_by(q, :os_version) do
defp group_imported_by(q, :os_version, _query) do
q
|> group_by([i], [i.operating_system, i.operating_system_version])
|> select_merge([i], %{
@ -640,19 +640,27 @@ defmodule Plausible.Stats.Imported do
})
end
defp group_imported_by(q, dim) when dim in [:entry_page, :exit_page] do
defp group_imported_by(q, dim, _query) when dim in [:entry_page, :exit_page] do
q
|> group_by([i], field(i, ^dim))
|> select_merge([i], %{^dim => field(i, ^dim)})
end
defp group_imported_by(q, :name) do
defp group_imported_by(q, :name, _query) do
q
|> group_by([i], i.name)
|> select_merge([i], %{name: i.name})
end
defp group_imported_by(q, :url) do
defp group_imported_by(q, :url, query) when query.v2 do
q
|> group_by([i], i.link_url)
|> select_merge([i], %{
url: fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)
})
end
defp group_imported_by(q, :url, _query) do
q
|> group_by([i], i.link_url)
|> select_merge([i], %{
@ -660,7 +668,15 @@ defmodule Plausible.Stats.Imported do
})
end
defp group_imported_by(q, :path) do
defp group_imported_by(q, :path, query) when query.v2 do
q
|> group_by([i], i.path)
|> select_merge([i], %{
path: fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)
})
end
defp group_imported_by(q, :path, _query) do
q
|> group_by([i], i.path)
|> select_merge([i], %{
@ -668,20 +684,20 @@ defmodule Plausible.Stats.Imported do
})
end
defp select_joined_dimension(q, :city) do
defp select_joined_dimension(q, :city, _query) do
select_merge(q, [s, i], %{
city: fragment("greatest(?,?)", i.city, s.city)
})
end
defp select_joined_dimension(q, :os_version) do
defp select_joined_dimension(q, :os_version, query) when not query.v2 do
select_merge(q, [s, i], %{
os: fragment("if(empty(?), ?, ?)", s.os, i.os, s.os),
os_version: fragment("if(empty(?), ?, ?)", s.os_version, i.os_version, s.os_version)
})
end
defp select_joined_dimension(q, :browser_version) do
defp select_joined_dimension(q, :browser_version, query) when not query.v2 do
select_merge(q, [s, i], %{
browser: fragment("if(empty(?), ?, ?)", s.browser, i.browser, s.browser),
browser_version:
@ -689,7 +705,7 @@ defmodule Plausible.Stats.Imported do
})
end
defp select_joined_dimension(q, dim) when dim in [:url, :path] do
defp select_joined_dimension(q, dim, query) when dim in [:url, :path] and not query.v2 do
select_merge(q, [s, i], %{
breakdown_prop_value:
fragment(
@ -701,7 +717,7 @@ defmodule Plausible.Stats.Imported do
})
end
defp select_joined_dimension(q, dim) do
defp select_joined_dimension(q, dim, _query) do
select_merge(q, [s, i], %{
^dim => fragment("if(empty(?), ?, ?)", field(s, ^dim), field(i, ^dim), field(s, ^dim))
})
@ -716,7 +732,7 @@ defmodule Plausible.Stats.Imported do
defp select_joined_metrics(q, [:visits | rest]) do
q
|> select_merge([s, i], %{visits: s.visits + i.visits})
|> select_merge([s, i], %{visits: selected_as(s.visits + i.visits, :visits)})
|> select_joined_metrics(rest)
end
@ -728,13 +744,13 @@ defmodule Plausible.Stats.Imported do
defp select_joined_metrics(q, [:events | rest]) do
q
|> select_merge([s, i], %{events: s.events + i.events})
|> select_merge([s, i], %{events: selected_as(s.events + i.events, :events)})
|> select_joined_metrics(rest)
end
defp select_joined_metrics(q, [:pageviews | rest]) do
q
|> select_merge([s, i], %{pageviews: s.pageviews + i.pageviews})
|> select_merge([s, i], %{pageviews: selected_as(s.pageviews + i.pageviews, :pageviews)})
|> select_joined_metrics(rest)
end

2
lib/plausible/stats/metrics.ex
View File

@ -21,6 +21,8 @@ defmodule Plausible.Stats.Metrics do
@metric_mappings Enum.into(@all_metrics, %{}, fn metric -> {to_string(metric), metric} end)
def metric?(value), do: Enum.member?(@all_metrics, value)
def from_string!(str) do
Map.fetch!(@metric_mappings, str)
end

19
lib/plausible/stats/query.ex
View File

@ -13,7 +13,11 @@ defmodule Plausible.Stats.Query do
now: nil,
experimental_session_count?: false,
experimental_reduced_joins?: false,
latest_import_end_date: nil
latest_import_end_date: nil,
metrics: [],
order_by: [],
timezone: nil,
v2: false
require OpenTelemetry.Tracer, as: Tracer
alias Plausible.Stats.{Filters, Interval, Imported}
@ -42,6 +46,19 @@ defmodule Plausible.Stats.Query do
query
end
def build(site, params) do
with {:ok, query_data} <- Filters.QueryParser.parse(site, params) do
query =
struct!(__MODULE__, Map.to_list(query_data))
|> put_imported_opts(site, %{})
|> put_experimental_session_count(site, params)
|> put_experimental_reduced_joins(site, params)
|> struct!(v2: true)
{:ok, query}
end
end
defp put_experimental_session_count(query, site, params) do
if Map.has_key?(params, "experimental_session_count") do
struct!(query,

28
lib/plausible/stats/query_optimizer.ex Normal file
View File

@ -0,0 +1,28 @@
defmodule Plausible.Stats.QueryOptimizer do
@moduledoc false
alias Plausible.Stats.Query
def optimize(query) do
Enum.reduce(pipeline(), query, fn step, acc -> step.(acc) end)
end
defp pipeline() do
[
&add_missing_order_by/1,
&update_group_by_time/1
]
end
defp add_missing_order_by(%Query{order_by: nil} = query) do
%Query{query | order_by: [{hd(query.metrics), :desc}]}
end
defp add_missing_order_by(query), do: query
defp update_group_by_time(%Query{dimensions: ["time" | rest]} = query) do
%Query{query | dimensions: ["time:month" | rest]}
end
defp update_group_by_time(query), do: query
end

52
lib/plausible/stats/query_result.ex Normal file
View File

@ -0,0 +1,52 @@
defmodule Plausible.Stats.QueryResult do
@moduledoc false
alias Plausible.Stats.SQL.QueryBuilder
alias Plausible.Stats.Filters
alias Plausible.Stats.Query
@derive Jason.Encoder
defstruct results: [],
query: nil,
meta: %{}
def from(results, query) do
results_list =
results
|> Enum.map(fn entry ->
%{
dimensions: Enum.map(query.dimensions, &Map.get(entry, QueryBuilder.shortname(&1))),
metrics: Enum.map(query.metrics, &Map.get(entry, &1))
}
end)
struct!(
__MODULE__,
results: results_list,
query: %{
metrics: query.metrics,
date_range: [query.date_range.first, query.date_range.last],
filters: query.filters |> Enum.map(&serializable_filter/1),
dimensions: query.dimensions,
order_by: query.order_by |> Enum.map(&Tuple.to_list/1)
},
meta: meta(query)
)
end
defp meta(%Query{skip_imported_reason: :unsupported_query}) do
%{
warning:
"Imported stats are not included in the results because query parameters are not supported. " <>
"For more information, see: https://plausible.io/docs/stats-api#filtering-imported-stats"
}
end
defp meta(_), do: %{}
defp serializable_filter([operation, "event:goal", clauses]) do
[operation, "event:goal", Enum.map(clauses, &Filters.Utils.unwrap_goal_value/1)]
end
defp serializable_filter(filter), do: filter
end

89
lib/plausible/stats/sql/expression.ex Normal file
View File

@ -0,0 +1,89 @@
defmodule Plausible.Stats.SQL.Expression do
@moduledoc false
import Ecto.Query
use Plausible.Stats.Fragments
@no_ref "Direct / None"
@not_set "(not set)"
defmacrop field_or_blank_value(expr, empty_value) do
quote do
dynamic(
[t],
fragment("if(empty(?), ?, ?)", unquote(expr), unquote(empty_value), unquote(expr))
)
end
end
def dimension("time:hour", query) do
dynamic([t], fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone))
end
def dimension("time:day", query) do
dynamic([t], fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone))
end
def dimension("time:month", query) do
dynamic([t], fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone))
end
def dimension("event:name", _query), do: dynamic([t], t.name)
def dimension("event:page", _query), do: dynamic([t], t.pathname)
def dimension("event:hostname", _query), do: dynamic([t], t.hostname)
def dimension("event:props:" <> property_name, _query) do
dynamic(
[t],
fragment(
"if(not empty(?), ?, '(none)')",
get_by_key(t, :meta, ^property_name),
get_by_key(t, :meta, ^property_name)
)
)
end
def dimension("visit:entry_page", _query), do: dynamic([t], t.entry_page)
def dimension("visit:exit_page", _query), do: dynamic([t], t.exit_page)
def dimension("visit:utm_medium", _query),
do: field_or_blank_value(t.utm_medium, @not_set)
def dimension("visit:utm_source", _query),
do: field_or_blank_value(t.utm_source, @not_set)
def dimension("visit:utm_campaign", _query),
do: field_or_blank_value(t.utm_campaign, @not_set)
def dimension("visit:utm_content", _query),
do: field_or_blank_value(t.utm_content, @not_set)
def dimension("visit:utm_term", _query),
do: field_or_blank_value(t.utm_term, @not_set)
def dimension("visit:source", _query),
do: field_or_blank_value(t.source, @no_ref)
def dimension("visit:referrer", _query),
do: field_or_blank_value(t.referrer, @no_ref)
def dimension("visit:device", _query),
do: field_or_blank_value(t.device, @not_set)
def dimension("visit:os", _query), do: field_or_blank_value(t.os, @not_set)
def dimension("visit:os_version", _query),
do: field_or_blank_value(t.os_version, @not_set)
def dimension("visit:browser", _query),
do: field_or_blank_value(t.browser, @not_set)
def dimension("visit:browser_version", _query),
do: field_or_blank_value(t.browser_version, @not_set)
# :TODO: Locations also set extra filters
def dimension("visit:country", _query), do: dynamic([t], t.country)
def dimension("visit:region", _query), do: dynamic([t], t.region)
def dimension("visit:city", _query), do: dynamic([t], t.city)
end

200
lib/plausible/stats/sql/query_builder.ex Normal file
View File

@ -0,0 +1,200 @@
defmodule Plausible.Stats.SQL.QueryBuilder do
@moduledoc false
use Plausible
import Ecto.Query
import Plausible.Stats.Imported
alias Plausible.Stats.{Base, Query, TableDecider, Util, Filters, Metrics}
alias Plausible.Stats.SQL.Expression
def build(query, site) do
{event_metrics, sessions_metrics, _other_metrics} =
query.metrics
|> Util.maybe_add_visitors_metric()
|> TableDecider.partition_metrics(query)
join_query_results(
build_events_query(site, query, event_metrics),
event_metrics,
build_sessions_query(site, query, sessions_metrics),
sessions_metrics,
query
)
end
def shortname(metric) when is_atom(metric), do: metric
def shortname(dimension), do: Plausible.Stats.Filters.without_prefix(dimension)
defp build_events_query(_, _, []), do: nil
defp build_events_query(site, query, event_metrics) do
q =
from(
e in "events_v2",
where: ^Filters.WhereBuilder.build(:events, site, query),
select: ^Base.select_event_metrics(event_metrics)
)
on_ee do
q = Plausible.Stats.Sampling.add_query_hint(q, query)
end
q
|> join_sessions_if_needed(site, query)
|> build_group_by(query)
|> merge_imported(site, query, event_metrics)
|> Base.maybe_add_conversion_rate(site, query, event_metrics)
end
defp join_sessions_if_needed(q, site, query) do
if TableDecider.events_join_sessions?(query) do
sessions_q =
from(
s in Base.query_sessions(site, query),
select: %{session_id: s.session_id},
where: s.sign == 1,
group_by: s.session_id
)
from(
e in q,
join: sq in subquery(sessions_q),
on: e.session_id == sq.session_id
)
else
q
end
end
def build_sessions_query(_, _, []), do: nil
def build_sessions_query(site, query, session_metrics) do
q =
from(
e in "sessions_v2",
where: ^Filters.WhereBuilder.build(:sessions, site, query),
select: ^Base.select_session_metrics(session_metrics, query)
)
on_ee do
q = Plausible.Stats.Sampling.add_query_hint(q, query)
end
q
|> join_events_if_needed(site, query)
|> build_group_by(query)
|> merge_imported(site, query, session_metrics)
end
def join_events_if_needed(q, site, query) do
if Query.has_event_filters?(query) do
events_q =
from(e in "events_v2",
where: ^Filters.WhereBuilder.build(:events, site, query),
select: %{
session_id: fragment("DISTINCT ?", e.session_id),
_sample_factor: fragment("_sample_factor")
}
)
on_ee do
events_q = Plausible.Stats.Sampling.add_query_hint(events_q, query)
end
from(s in q,
join: e in subquery(events_q),
on: s.session_id == e.session_id
)
else
q
end
end
defp build_group_by(q, query) do
Enum.reduce(query.dimensions, q, fn dimension, q ->
q
|> select_merge(^%{shortname(dimension) => Expression.dimension(dimension, query)})
|> group_by(^Expression.dimension(dimension, query))
end)
end
defp build_order_by(q, query, mode) do
Enum.reduce(query.order_by, q, &build_order_by(&2, query, &1, mode))
end
def build_order_by(q, query, {metric_or_dimension, order_direction}, :inner) do
order_by(
q,
[t],
^{
order_direction,
if(
Metrics.metric?(metric_or_dimension),
do: dynamic([], selected_as(^shortname(metric_or_dimension))),
else: Expression.dimension(metric_or_dimension, query)
)
}
)
end
def build_order_by(q, _query, {metric_or_dimension, order_direction}, :outer) do
order_by(
q,
[t],
^{
order_direction,
dynamic([], selected_as(^shortname(metric_or_dimension)))
}
)
end
defmacrop select_join_fields(q, list, table_name) do
quote do
Enum.reduce(unquote(list), unquote(q), fn metric_or_dimension, q ->
select_merge(
q,
^%{
shortname(metric_or_dimension) =>
dynamic(
[e, s],
selected_as(
field(unquote(table_name), ^shortname(metric_or_dimension)),
^shortname(metric_or_dimension)
)
)
}
)
end)
end
end
defp join_query_results(nil, _, nil, _, _query), do: nil
defp join_query_results(events_q, _, nil, _, query),
do: events_q |> build_order_by(query, :inner)
defp join_query_results(nil, _, sessions_q, _, query),
do: sessions_q |> build_order_by(query, :inner)
defp join_query_results(events_q, event_metrics, sessions_q, sessions_metrics, query) do
join(subquery(events_q), :left, [e], s in subquery(sessions_q),
on: ^build_group_by_join(query)
)
|> select_join_fields(query.dimensions, e)
|> select_join_fields(event_metrics, e)
|> select_join_fields(List.delete(sessions_metrics, :sample_percent), s)
|> build_order_by(query, :outer)
end
defp build_group_by_join(%Query{dimensions: []}), do: true
defp build_group_by_join(query) do
query.dimensions
|> Enum.map(fn dim ->
dynamic([e, s], field(e, ^shortname(dim)) == field(s, ^shortname(dim)))
end)
|> Enum.reduce(fn condition, acc -> dynamic([], ^acc and ^condition) end)
end
end

44
lib/plausible/stats/table_decider.ex
View File

@ -9,10 +9,18 @@ defmodule Plausible.Stats.TableDecider do
alias Plausible.Stats.Query
def events_join_sessions?(query) do
Enum.any?(query.filters, &(filters_partitioner(query, &1) == :session))
query
|> filter_keys()
|> Enum.any?(&(filters_partitioner(query, &1) == :session))
end
def partition_metrics(metrics, query, breakdown_property \\ nil) do
def event_filters?(query) do
query
|> filter_keys()
|> Enum.any?(&(filters_partitioner(query, &1) == :event))
end
def partition_metrics(metrics, query) do
%{
event: event_only_metrics,
session: session_only_metrics,
@ -22,16 +30,10 @@ defmodule Plausible.Stats.TableDecider do
} =
partition(metrics, query, &metric_partitioner/2)
# Treat breakdown property as yet another filter
query =
if breakdown_property do
Query.put_filter(query, [:is, breakdown_property, []])
else
query
end
%{event: event_only_filters, session: session_only_filters} =
partition(query.filters, query, &filters_partitioner/2)
query
|> filter_keys()
|> partition(query, &filters_partitioner/2)
cond do
# Only one table needs to be queried
@ -55,6 +57,12 @@ defmodule Plausible.Stats.TableDecider do
end
end
defp filter_keys(query) do
query.filters
|> Enum.map(fn [_, filter_key | _rest] -> filter_key end)
|> Enum.concat(query.dimensions)
end
defp metric_partitioner(_, :conversion_rate), do: :event
defp metric_partitioner(_, :average_revenue), do: :event
defp metric_partitioner(_, :total_revenue), do: :event
@ -83,16 +91,16 @@ defmodule Plausible.Stats.TableDecider do
defp metric_partitioner(_, _), do: :either
defp filters_partitioner(_, [_, "event:" <> _ | _rest]), do: :event
defp filters_partitioner(_, [_, "visit:entry_page" | _rest]), do: :session
defp filters_partitioner(_, [_, "visit:entry_page_hostname" | _rest]), do: :session
defp filters_partitioner(_, [_, "visit:exit_page" | _rest]), do: :session
defp filters_partitioner(_, [_, "visit:exit_page_hostname" | _rest]), do: :session
defp filters_partitioner(_, "event:" <> _), do: :event
defp filters_partitioner(_, "visit:entry_page"), do: :session
defp filters_partitioner(_, "visit:entry_page_hostname"), do: :session
defp filters_partitioner(_, "visit:exit_page"), do: :session
defp filters_partitioner(_, "visit:exit_page_hostname"), do: :session
defp filters_partitioner(%Query{experimental_reduced_joins?: true}, [_, "visit:" <> _ | _rest]),
defp filters_partitioner(%Query{experimental_reduced_joins?: true}, "visit:" <> _),
do: :either
defp filters_partitioner(_, [_, "visit:" <> _ | _rest]),
defp filters_partitioner(_, "visit:" <> _),
do: :session
defp filters_partitioner(%Query{experimental_reduced_joins?: false}, {unknown, _}) do

23
lib/plausible_web/controllers/api/external_query_api_controller.ex Normal file
View File

@ -0,0 +1,23 @@
defmodule PlausibleWeb.Api.ExternalQueryApiController do
@moduledoc false
use PlausibleWeb, :controller
use Plausible.Repo
use PlausibleWeb.Plugs.ErrorHandler
alias Plausible.Stats.Query
def query(conn, params) do
site = Repo.preload(conn.assigns.site, :owner)
case Query.build(site, params) do
{:ok, query} ->
results = Plausible.Stats.query(site, query)
json(conn, results)
{:error, message} ->
conn
|> put_status(400)
|> json(%{error: message})
end
end
end

6
lib/plausible_web/router.ex
View File

@ -175,6 +175,12 @@ defmodule PlausibleWeb.Router do
get "/timeseries", ExternalStatsController, :timeseries
end
scope "/api/v2", PlausibleWeb.Api do
pipe_through [:public_api, PlausibleWeb.AuthorizeStatsApiPlug]
post "/query", ExternalQueryApiController, :query
end
on_ee do
scope "/api/v1/sites", PlausibleWeb.Api do
pipe_through [:public_api, PlausibleWeb.AuthorizeSitesApiPlug]

417
test/plausible/stats/query_parser_test.exs
View File

@ -1,44 +1,50 @@
defmodule Plausible.Stats.Filters.QueryParserTest do
use ExUnit.Case, async: true
use Plausible.DataCase
alias Plausible.Stats.Filters
import Plausible.Stats.Filters.QueryParser
def check_success(params, expected_result) do
assert parse(params) == {:ok, expected_result}
setup [:create_user, :create_new_site]
@date_range Date.range(Timex.today(), Timex.today())
def check_success(params, site, expected_result) do
assert parse(site, params) == {:ok, expected_result}
end
def check_error(params, expected_error_message) do
{:error, message} = parse(params)
def check_error(params, site, expected_error_message) do
{:error, message} = parse(site, params)
assert message =~ expected_error_message
end
test "parsing empty map fails" do
test "parsing empty map fails", %{site: site} do
%{}
|> check_error("No valid metrics passed")
|> check_error(site, "No valid metrics passed")
end
describe "metrics validation" do
test "valid metrics passed" do
test "valid metrics passed", %{site: site} do
%{"metrics" => ["visitors", "events"], "date_range" => "all"}
|> check_success(%{
|> check_success(site, %{
metrics: [:visitors, :events],
date_range: "all",
date_range: @date_range,
filters: [],
dimensions: [],
order_by: nil
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
})
end
test "invalid metric passed" do
test "invalid metric passed", %{site: site} do
%{"metrics" => ["visitors", "event:name"], "date_range" => "all"}
|> check_error("Unknown metric '\"event:name\"'")
|> check_error(site, "Unknown metric '\"event:name\"'")
end
test "fuller list of metrics" do
test "fuller list of metrics", %{site: site} do
%{
"metrics" => [
"time_on_page",
"conversion_rate",
"visitors",
"pageviews",
"visits",
@ -48,10 +54,9 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
],
"date_range" => "all"
}
|> check_success(%{
|> check_success(site, %{
metrics: [
:time_on_page,
:conversion_rate,
:visitors,
:pageviews,
:visits,
@ -59,22 +64,24 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
:bounce_rate,
:visit_duration
],
date_range: "all",
date_range: @date_range,
filters: [],
dimensions: [],
order_by: nil
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
})
end
test "same metric queried multiple times" do
test "same metric queried multiple times", %{site: site} do
%{"metrics" => ["events", "visitors", "visitors"], "date_range" => "all"}
|> check_error(~r/Metrics cannot be queried multiple times/)
|> check_error(site, ~r/Metrics cannot be queried multiple times/)
end
end
describe "filters validation" do
for operation <- [:is, :is_not, :matches, :does_not_match] do
test "#{operation} filter" do
test "#{operation} filter", %{site: site} do
%{
"metrics" => ["visitors"],
"date_range" => "all",
@ -82,18 +89,20 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
[Atom.to_string(unquote(operation)), "event:name", ["foo"]]
]
}
|> check_success(%{
|> check_success(site, %{
metrics: [:visitors],
date_range: "all",
date_range: @date_range,
filters: [
[unquote(operation), "event:name", ["foo"]]
],
dimensions: [],
order_by: nil
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
})
end
test "#{operation} filter with invalid clause" do
test "#{operation} filter with invalid clause", %{site: site} do
%{
"metrics" => ["visitors"],
"date_range" => "all",
@ -101,11 +110,11 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
[Atom.to_string(unquote(operation)), "event:name", "foo"]
]
}
|> check_error(~r/Invalid filter/)
|> check_error(site, ~r/Invalid filter/)
end
end
test "filtering by invalid operation" do
test "filtering by invalid operation", %{site: site} do
%{
"metrics" => ["visitors"],
"date_range" => "all",
@ -113,10 +122,10 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
["exists?", "event:name", ["foo"]]
]
}
|> check_error(~r/Unknown operator for filter/)
|> check_error(site, ~r/Unknown operator for filter/)
end
test "filtering by custom properties" do
test "filtering by custom properties", %{site: site} do
%{
"metrics" => ["visitors"],
"date_range" => "all",
@ -124,20 +133,22 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
["is", "event:props:foobar", ["value"]]
]
}
|> check_success(%{
|> check_success(site, %{
metrics: [:visitors],
date_range: "all",
date_range: @date_range,
filters: [
[:is, "event:props:foobar", ["value"]]
],
dimensions: [],
order_by: nil
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
})
end
for dimension <- Filters.event_props() do
if dimension != "goal" do
test "filtering by event:#{dimension} filter" do
test "filtering by event:#{dimension} filter", %{site: site} do
%{
"metrics" => ["visitors"],
"date_range" => "all",
@ -145,21 +156,23 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
["is", "event:#{unquote(dimension)}", ["foo"]]
]
}
|> check_success(%{
|> check_success(site, %{
metrics: [:visitors],
date_range: "all",
date_range: @date_range,
filters: [
[:is, "event:#{unquote(dimension)}", ["foo"]]
],
dimensions: [],
order_by: nil
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
})
end
end
end
for dimension <- Filters.visit_props() do
test "filtering by visit:#{dimension} filter" do
test "filtering by visit:#{dimension} filter", %{site: site} do
%{
"metrics" => ["visitors"],
"date_range" => "all",
@ -167,38 +180,21 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
["is", "visit:#{unquote(dimension)}", ["foo"]]
]
}
|> check_success(%{
|> check_success(site, %{
metrics: [:visitors],
date_range: "all",
date_range: @date_range,
filters: [
[:is, "visit:#{unquote(dimension)}", ["foo"]]
],
dimensions: [],
order_by: nil
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
})
end
end
test "filtering by event:goal" do
%{
"metrics" => ["visitors"],
"date_range" => "all",
"filters" => [
["is", "event:goal", ["Signup", "Visit /thank-you"]]
]
}
|> check_success(%{
metrics: [:visitors],
date_range: "all",
filters: [
[:is, "event:goal", [{:event, "Signup"}, {:page, "/thank-you"}]]
],
dimensions: [],
order_by: nil
})
end
test "invalid event filter" do
test "invalid event filter", %{site: site} do
%{
"metrics" => ["visitors"],
"date_range" => "all",
@ -206,10 +202,10 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
["is", "event:device", ["foo"]]
]
}
|> check_error(~r/Invalid filter /)
|> check_error(site, ~r/Invalid filter /)
end
test "invalid visit filter" do
test "invalid visit filter", %{site: site} do
%{
"metrics" => ["visitors"],
"date_range" => "all",
@ -217,16 +213,92 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
["is", "visit:name", ["foo"]]
]
}
|> check_error(~r/Invalid filter /)
|> check_error(site, ~r/Invalid filter /)
end
test "invalid filter" do
test "invalid filter", %{site: site} do
%{
"metrics" => ["visitors"],
"date_range" => "all",
"filters" => "foobar"
}
|> check_error(~r/Invalid filters passed/)
|> check_error(site, ~r/Invalid filters passed/)
end
end
describe "include validation" do
test "setting include.imports", %{site: site} do
%{
"metrics" => ["visitors"],
"date_range" => "all",
"include" => %{"imports" => true}
}
|> check_success(site, %{
metrics: [:visitors],
date_range: @date_range,
filters: [],
dimensions: [],
order_by: nil,
timezone: site.timezone,
imported_data_requested: true
})
end
test "setting invalid imports value", %{site: site} do
%{
"metrics" => ["visitors"],
"date_range" => "all",
"include" => "foobar"
}
|> check_error(site, ~r/Invalid include passed/)
end
end
describe "event:goal filter validation" do
test "valid filters", %{site: site} do
insert(:goal, %{site: site, event_name: "Signup"})
insert(:goal, %{site: site, page_path: "/thank-you"})
%{
"metrics" => ["visitors"],
"date_range" => "all",
"filters" => [
["is", "event:goal", ["Signup", "Visit /thank-you"]]
]
}
|> check_success(site, %{
metrics: [:visitors],
date_range: @date_range,
filters: [
[:is, "event:goal", [{:event, "Signup"}, {:page, "/thank-you"}]]
],
dimensions: [],
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
})
end
test "invalid event filter", %{site: site} do
%{
"metrics" => ["visitors"],
"date_range" => "all",
"filters" => [
["is", "event:goal", ["Signup"]]
]
}
|> check_error(site, ~r/The goal `Signup` is not configured for this site/)
end
test "invalid page filter", %{site: site} do
%{
"metrics" => ["visitors"],
"date_range" => "all",
"filters" => [
["is", "event:goal", ["Visit /thank-you"]]
]
}
|> check_error(site, ~r/The goal `Visit \/thank-you` is not configured for this site/)
end
end
@ -235,139 +307,288 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
describe "dimensions validation" do
for dimension <- Filters.event_props() do
test "event:#{dimension} dimension" do
test "event:#{dimension} dimension", %{site: site} do
%{
"metrics" => ["visitors"],
"date_range" => "all",
"dimensions" => ["event:#{unquote(dimension)}"]
}
|> check_success(%{
|> check_success(site, %{
metrics: [:visitors],
date_range: "all",
date_range: @date_range,
filters: [],
dimensions: ["event:#{unquote(dimension)}"],
order_by: nil
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
})
end
end
for dimension <- Filters.visit_props() do
test "visit:#{dimension} dimension" do
test "visit:#{dimension} dimension", %{site: site} do
%{
"metrics" => ["visitors"],
"date_range" => "all",
"dimensions" => ["visit:#{unquote(dimension)}"]
}
|> check_success(%{
|> check_success(site, %{
metrics: [:visitors],
date_range: "all",
date_range: @date_range,
filters: [],
dimensions: ["visit:#{unquote(dimension)}"],
order_by: nil
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
})
end
end
test "custom properties dimension" do
test "custom properties dimension", %{site: site} do
%{
"metrics" => ["visitors"],
"date_range" => "all",
"dimensions" => ["event:props:foobar"]
}
|> check_success(%{
|> check_success(site, %{
metrics: [:visitors],
date_range: "all",
date_range: @date_range,
filters: [],
dimensions: ["event:props:foobar"],
order_by: nil
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
})
end
test "invalid dimension name passed" do
test "invalid custom property dimension", %{site: site} do
%{
"metrics" => ["visitors"],
"date_range" => "all",
"dimensions" => ["event:props:"]
}
|> check_error(site, ~r/Invalid dimensions/)
end
test "invalid dimension name passed", %{site: site} do
%{
"metrics" => ["visitors"],
"date_range" => "all",
"dimensions" => ["visitors"]
}
|> check_error(~r/Invalid dimensions/)
|> check_error(site, ~r/Invalid dimensions/)
end
test "invalid dimension" do
test "invalid dimension", %{site: site} do
%{
"metrics" => ["visitors"],
"date_range" => "all",
"dimensions" => "foobar"
}
|> check_error(~r/Invalid dimensions/)
|> check_error(site, ~r/Invalid dimensions/)
end
test "dimensions are not unique" do
test "dimensions are not unique", %{site: site} do
%{
"metrics" => ["visitors"],
"date_range" => "all",
"dimensions" => ["event:name", "event:name"]
}
|> check_error(~r/Some dimensions are listed multiple times/)
|> check_error(site, ~r/Some dimensions are listed multiple times/)
end
end
describe "order_by validation" do
test "ordering by metric" do
test "ordering by metric", %{site: site} do
%{
"metrics" => ["visitors", "events"],
"date_range" => "all",
"order_by" => [["events", "desc"], ["visitors", "asc"]]
}
|> check_success(%{
|> check_success(site, %{
metrics: [:visitors, :events],
date_range: "all",
date_range: @date_range,
filters: [],
dimensions: [],
order_by: [{:events, :desc}, {:visitors, :asc}]
order_by: [{:events, :desc}, {:visitors, :asc}],
timezone: site.timezone,
imported_data_requested: false
})
end
test "ordering by dimension" do
test "ordering by dimension", %{site: site} do
%{
"metrics" => ["visitors"],
"date_range" => "all",
"dimensions" => ["event:name"],
"order_by" => [["event:name", "desc"]]
}
|> check_success(%{
|> check_success(site, %{
metrics: [:visitors],
date_range: "all",
date_range: @date_range,
filters: [],
dimensions: ["event:name"],
order_by: [{"event:name", :desc}]
order_by: [{"event:name", :desc}],
timezone: site.timezone,
imported_data_requested: false
})
end
test "ordering by invalid value" do
test "ordering by invalid value", %{site: site} do
%{
"metrics" => ["visitors"],
"date_range" => "all",
"order_by" => [["visssss", "desc"]]
}
|> check_error(~r/Invalid order_by entry/)
|> check_error(site, ~r/Invalid order_by entry/)
end
test "ordering by not queried metric" do
test "ordering by not queried metric", %{site: site} do
%{
"metrics" => ["visitors"],
"date_range" => "all",
"order_by" => [["events", "desc"]]
}
|> check_error(~r/Entry is not a queried metric or dimension/)
|> check_error(site, ~r/Entry is not a queried metric or dimension/)
end
test "ordering by not queried dimension" do
test "ordering by not queried dimension", %{site: site} do
%{
"metrics" => ["visitors"],
"date_range" => "all",
"order_by" => [["event:name", "desc"]]
}
|> check_error(~r/Entry is not a queried metric or dimension/)
|> check_error(site, ~r/Entry is not a queried metric or dimension/)
end
end
describe "custom props access" do
test "error if invalid filter", %{site: site, user: user} do
ep =
insert(:enterprise_plan, features: [Plausible.Billing.Feature.StatsAPI], user_id: user.id)
insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)
%{
"metrics" => ["visitors"],
"date_range" => "all",
"filters" => [["is", "event:props:foobar", ["foo"]]]
}
|> check_error(
site,
~r/The owner of this site does not have access to the custom properties feature/
)
end
test "error if invalid dimension", %{site: site, user: user} do
ep =
insert(:enterprise_plan, features: [Plausible.Billing.Feature.StatsAPI], user_id: user.id)
insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)
%{
"metrics" => ["visitors"],
"date_range" => "all",
"dimensions" => ["event:props:foobar"]
}
|> check_error(
site,
~r/The owner of this site does not have access to the custom properties feature/
)
end
end
describe "conversion_rate metric" do
test "fails validation on its own", %{site: site} do
%{
"metrics" => ["conversion_rate"],
"date_range" => "all"
}
|> check_error(
site,
~r/Metric `conversion_rate` can only be queried with event:goal filters or dimensions/
)
end
test "succeeds with event:goal filter", %{site: site} do
insert(:goal, %{site: site, event_name: "Signup"})
%{
"metrics" => ["conversion_rate"],
"date_range" => "all",
"filters" => [["is", "event:goal", ["Signup"]]]
}
|> check_success(site, %{
metrics: [:conversion_rate],
date_range: @date_range,
filters: [[:is, "event:goal", [event: "Signup"]]],
dimensions: [],
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
})
end
test "succeeds with event:goal dimension", %{site: site} do
insert(:goal, %{site: site, event_name: "Signup"})
%{
"metrics" => ["conversion_rate"],
"date_range" => "all",
"dimensions" => ["event:goal"]
}
|> check_success(site, %{
metrics: [:conversion_rate],
date_range: @date_range,
filters: [],
dimensions: ["event:goal"],
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
})
end
end
describe "session metrics" do
test "single session metric succeeds", %{site: site} do
%{
"metrics" => ["bounce_rate"],
"date_range" => "all",
"dimensions" => ["visit:device"]
}
|> check_success(site, %{
metrics: [:bounce_rate],
date_range: @date_range,
filters: [],
dimensions: ["visit:device"],
order_by: nil,
timezone: site.timezone,
imported_data_requested: false
})
end
test "fails if using session metric with event dimension", %{site: site} do
%{
"metrics" => ["bounce_rate"],
"date_range" => "all",
"dimensions" => ["event:props:foo"]
}
|> check_error(
site,
"Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions"
)
end
test "fails if using session metric with event filter", %{site: site} do
%{
"metrics" => ["bounce_rate"],
"date_range" => "all",
"filters" => [["is", "event:props:foo", ["(none)"]]]
}
|> check_error(
site,
"Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions"
)
end
end
end

72
test/plausible/stats/table_decider_test.exs
View File

@ -5,62 +5,62 @@ defmodule Plausible.Stats.TableDeciderTest do
import Plausible.Stats.TableDecider
test "events_join_sessions? with experimental_reduced_joins disabled" do
assert not events_join_sessions?(make_query(false, %{}))
assert not events_join_sessions?(make_query(false, %{name: "pageview"}))
assert events_join_sessions?(make_query(false, %{source: "Google"}))
assert events_join_sessions?(make_query(false, %{entry_page: "/"}))
assert events_join_sessions?(make_query(false, %{exit_page: "/"}))
assert not events_join_sessions?(make_query(false, []))
assert not events_join_sessions?(make_query(false, ["event:name"]))
assert events_join_sessions?(make_query(false, ["visit:source"]))
assert events_join_sessions?(make_query(false, ["visit:entry_page"]))
assert events_join_sessions?(make_query(false, ["visit:exit_page"]))
end
test "events_join_sessions? with experimental_reduced_joins enabled" do
assert not events_join_sessions?(make_query(true, %{}))
assert not events_join_sessions?(make_query(true, %{name: "pageview"}))
assert not events_join_sessions?(make_query(true, %{source: "Google"}))
assert events_join_sessions?(make_query(true, %{entry_page: "/"}))
assert events_join_sessions?(make_query(true, %{exit_page: "/"}))
assert not events_join_sessions?(make_query(true, []))
assert not events_join_sessions?(make_query(true, ["event:name"]))
assert not events_join_sessions?(make_query(true, ["visit:source"]))
assert events_join_sessions?(make_query(true, ["visit:entry_page"]))
assert events_join_sessions?(make_query(true, ["visit:exit_page"]))
end
describe "partition_metrics" do
test "with no metrics or filters" do
query = make_query(false, %{})
query = make_query(false, [])
assert partition_metrics([], query) == {[], [], []}
end
test "session-only metrics accordingly" do
query = make_query(false, %{})
query = make_query(false, [])
assert partition_metrics([:bounce_rate, :views_per_visit], query) ==
{[], [:bounce_rate, :views_per_visit], []}
end
test "event-only metrics accordingly" do
query = make_query(false, %{})
query = make_query(false, [])
assert partition_metrics([:total_revenue, :visitors], query) ==
{[:total_revenue, :visitors], [], []}
end
test "filters from both, event-only metrics" do
query = make_query(false, %{name: "pageview", source: "Google"})
query = make_query(false, ["event:name", "visit:source"])
assert partition_metrics([:total_revenue], query) == {[:total_revenue], [], []}
end
test "filters from both, session-only metrics" do
query = make_query(false, %{name: "pageview", source: "Google"})
query = make_query(false, ["event:name", "visit:source"])
assert partition_metrics([:bounce_rate], query) == {[], [:bounce_rate], []}
end
test "session filters but no session metrics" do
query = make_query(false, %{source: "Google"})
query = make_query(false, ["visit:source"])
assert partition_metrics([:total_revenue], query) == {[:total_revenue], [], []}
end
test "sample_percent is added to both types of metrics" do
query = make_query(false, %{})
query = make_query(false, [])
assert partition_metrics([:total_revenue, :sample_percent], query) ==
{[:total_revenue, :sample_percent], [], []}
@ -73,14 +73,14 @@ defmodule Plausible.Stats.TableDeciderTest do
end
test "other metrics put in its own result" do
query = make_query(false, %{})
query = make_query(false, [])
assert partition_metrics([:time_on_page, :percentage, :total_visitors], query) ==
{[], [], [:time_on_page, :percentage, :total_visitors]}
end
test "raises if unknown metric" do
query = make_query(false, %{})
query = make_query(false, [])
assert_raise ArgumentError, fn ->
partition_metrics([:foobar], query)
@ -90,7 +90,7 @@ defmodule Plausible.Stats.TableDeciderTest do
describe "partition_metrics with experimental_reduced_joins enabled" do
test "metrics that can be calculated on either when event-only metrics" do
query = make_query(true, %{})
query = make_query(true, [])
assert partition_metrics([:total_revenue, :visitors], query) ==
{[:total_revenue, :visitors], [], []}
@ -99,7 +99,7 @@ defmodule Plausible.Stats.TableDeciderTest do
end
test "metrics that can be calculated on either when session-only metrics" do
query = make_query(true, %{})
query = make_query(true, [])
assert partition_metrics([:bounce_rate, :visitors], query) ==
{[], [:bounce_rate, :visitors], []}
@ -109,56 +109,60 @@ defmodule Plausible.Stats.TableDeciderTest do
end
test "metrics that can be calculated on either are biased to sessions" do
query = make_query(true, %{})
query = make_query(true, [])
assert partition_metrics([:bounce_rate, :total_revenue, :visitors], query) ==
{[:total_revenue], [:bounce_rate, :visitors], []}
end
test "sample_percent is handled with either metrics" do
query = make_query(true, %{})
query = make_query(true, [])
assert partition_metrics([:visitors, :sample_percent], query) ==
{[], [:visitors, :sample_percent], []}
end
test "metric can be calculated on either, but filtering on events" do
query = make_query(true, %{name: "pageview"})
query = make_query(true, ["event:name"])
assert partition_metrics([:visitors], query) == {[:visitors], [], []}
end
test "metric can be calculated on either, but filtering on events and sessions" do
query = make_query(true, %{name: "pageview", exit_page: "/"})
query = make_query(true, ["event:name", "visit:exit_page"])
assert partition_metrics([:visitors], query) == {[], [:visitors], []}
end
test "metric can be calculated on either, filtering on either" do
query = make_query(true, %{source: "Google"})
query = make_query(true, ["visit:source"])
assert partition_metrics([:visitors], query) == {[], [:visitors], []}
end
test "metric can be calculated on either, filtering on sessions" do
query = make_query(true, %{exit_page: "/"})
query = make_query(true, ["visit:exit_page"])
assert partition_metrics([:visitors], query) == {[], [:visitors], []}
end
test "breakdown value leans metric" do
query = make_query(true, %{})
test "query dimensions lean metric" do
assert partition_metrics([:visitors], make_query(true, [], ["event:name"])) ==
{[:visitors], [], []}
assert partition_metrics([:visitors], query, "event:name") == {[:visitors], [], []}
assert partition_metrics([:visitors], query, "visit:source") == {[], [:visitors], []}
assert partition_metrics([:visitors], query, "visit:exit_page") == {[], [:visitors], []}
assert partition_metrics([:visitors], make_query(true, [], ["visit:source"])) ==
{[], [:visitors], []}
assert partition_metrics([:visitors], make_query(true, [], ["visit:exit_page"])) ==
{[], [:visitors], []}
end
end
defp make_query(experimental_reduced_joins?, filters) do
defp make_query(experimental_reduced_joins?, filter_keys, dimensions \\ []) do
Query.from(build(:site), %{
"experimental_reduced_joins" => to_string(experimental_reduced_joins?),
"filters" => Jason.encode!(filters)
"filters" => Enum.map(filter_keys, fn filter_key -> ["is", filter_key, []] end),
"dimensions" => dimensions
})
end
end

3106
test/plausible_web/controllers/api/external_stats_controller/query_test.exs Normal file
View File

File diff suppressed because it is too large Load Diff