mirror of
https://github.com/plausible/analytics.git
synced 2024-08-17 05:40:24 +03:00
APIv2 - initial PR (#4216)
* WIP new querying * WIP: Move some aggregate code under new command * WIP: Add joins, handling less metrics * join events table to sessions if needed * Merge imported results with built query * Remove dead code * WIP: /api/v2/query * Allow grouping by time * Use JOIN for main query * Build query result * update parse_time * Make joinless order by work * First test * more breakdown tests * Serialize event:goal filters in an json-encodable way/reflection * Handle inner vs outer ORDER BY clauses properly * Handle single conversion_rate metric * Update more tests * Get parsing tests passing again * Validate filtered goal filter is configured * Enable more validation tests * Enable more event:name breakdown tests * Enable more breakdown tests * Validate site has access to custom props * Validate conversion_rate metric which is only allowed in some situations * Validate that empty event:props: is not valid * handle query.dimensions properly in table_decider * test more validations on metrics/dimensions * Validate session metrics in combination with event dimension(s) * Tests cleanup * Parse include.imports * Get imports working with new querying * Make more imports tests work * Make event:props:path imports-adjacent test work * Get query imports warning-related tests running * Remove dead pagination tests * Solve dead import * Solve some warnings * Update aggregate metrics tests * credo * Improve test naming * Lazy goal loading * Use datetime methods * Ecto -> SQL module name * Remove Expression.dimension mode option
This commit is contained in:
parent
5382020ff0
commit
58a66a952c
@ -15,14 +15,20 @@ defmodule Plausible.Stats.Goal.Revenue do
|
||||
def total_revenue_query() do
|
||||
dynamic(
|
||||
[e],
|
||||
fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)
|
||||
selected_as(
|
||||
fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount),
|
||||
:total_revenue
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
def average_revenue_query() do
|
||||
dynamic(
|
||||
[e],
|
||||
fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)
|
||||
selected_as(
|
||||
fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount),
|
||||
:average_revenue
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
|
@ -1,12 +1,16 @@
|
||||
defmodule Plausible.Stats do
|
||||
use Plausible
|
||||
alias Plausible.Stats.QueryResult
|
||||
use Plausible.ClickhouseRepo
|
||||
|
||||
alias Plausible.Stats.{
|
||||
Breakdown,
|
||||
Aggregate,
|
||||
Timeseries,
|
||||
CurrentVisitors,
|
||||
FilterSuggestions
|
||||
FilterSuggestions,
|
||||
QueryOptimizer,
|
||||
SQL
|
||||
}
|
||||
|
||||
use Plausible.DebugReplayInfo
|
||||
@ -31,6 +35,15 @@ defmodule Plausible.Stats do
|
||||
CurrentVisitors.current_visitors(site)
|
||||
end
|
||||
|
||||
def query(site, query) do
|
||||
optimized_query = QueryOptimizer.optimize(query)
|
||||
|
||||
optimized_query
|
||||
|> SQL.QueryBuilder.build(site)
|
||||
|> ClickhouseRepo.all()
|
||||
|> QueryResult.from(optimized_query)
|
||||
end
|
||||
|
||||
on_ee do
|
||||
def funnel(site, query, funnel) do
|
||||
include_sentry_replay_info()
|
||||
|
@ -1,7 +1,7 @@
|
||||
defmodule Plausible.Stats.Aggregate do
|
||||
use Plausible.ClickhouseRepo
|
||||
use Plausible
|
||||
import Plausible.Stats.{Base, Imported}
|
||||
import Plausible.Stats.Base
|
||||
import Ecto.Query
|
||||
alias Plausible.Stats.{Query, Util}
|
||||
|
||||
@ -15,23 +15,24 @@ defmodule Plausible.Stats.Aggregate do
|
||||
|
||||
Query.trace(query, metrics)
|
||||
|
||||
{event_metrics, session_metrics, other_metrics} =
|
||||
metrics
|
||||
|> Util.maybe_add_visitors_metric()
|
||||
|> Plausible.Stats.TableDecider.partition_metrics(query)
|
||||
query_with_metrics = %Plausible.Stats.Query{
|
||||
query
|
||||
| metrics: Util.maybe_add_visitors_metric(metrics)
|
||||
}
|
||||
|
||||
event_task = fn -> aggregate_events(site, query, event_metrics) end
|
||||
|
||||
session_task = fn -> aggregate_sessions(site, query, session_metrics) end
|
||||
q = Plausible.Stats.SQL.QueryBuilder.build(query_with_metrics, site)
|
||||
|
||||
time_on_page_task =
|
||||
if :time_on_page in other_metrics do
|
||||
if :time_on_page in query_with_metrics.metrics do
|
||||
fn -> aggregate_time_on_page(site, query) end
|
||||
else
|
||||
fn -> %{} end
|
||||
end
|
||||
|
||||
Plausible.ClickhouseRepo.parallel_tasks([session_task, event_task, time_on_page_task])
|
||||
Plausible.ClickhouseRepo.parallel_tasks([
|
||||
run_query_task(q),
|
||||
time_on_page_task
|
||||
])
|
||||
|> Enum.reduce(%{}, fn aggregate, task_result -> Map.merge(aggregate, task_result) end)
|
||||
|> Util.keep_requested_metrics(metrics)
|
||||
|> cast_revenue_metrics_to_money(currency)
|
||||
@ -40,24 +41,8 @@ defmodule Plausible.Stats.Aggregate do
|
||||
|> Enum.into(%{})
|
||||
end
|
||||
|
||||
defp aggregate_events(_, _, []), do: %{}
|
||||
|
||||
defp aggregate_events(site, query, metrics) do
|
||||
from(e in base_event_query(site, query), select: ^select_event_metrics(metrics))
|
||||
|> merge_imported(site, query, metrics)
|
||||
|> maybe_add_conversion_rate(site, query, metrics)
|
||||
|> ClickhouseRepo.one()
|
||||
end
|
||||
|
||||
defp aggregate_sessions(_, _, []), do: %{}
|
||||
|
||||
defp aggregate_sessions(site, query, metrics) do
|
||||
from(e in query_sessions(site, query), select: ^select_session_metrics(metrics, query))
|
||||
|> filter_converted_sessions(site, query)
|
||||
|> merge_imported(site, query, metrics)
|
||||
|> ClickhouseRepo.one()
|
||||
|> Util.keep_requested_metrics(metrics)
|
||||
end
|
||||
defp run_query_task(nil), do: fn -> %{} end
|
||||
defp run_query_task(q), do: fn -> ClickhouseRepo.one(q) end
|
||||
|
||||
defp aggregate_time_on_page(site, query) do
|
||||
windowed_pages_q =
|
||||
|
@ -31,7 +31,7 @@ defmodule Plausible.Stats.Base do
|
||||
end
|
||||
end
|
||||
|
||||
defp query_events(site, query) do
|
||||
def query_events(site, query) do
|
||||
q = from(e in "events_v2", where: ^Filters.WhereBuilder.build(:events, site, query))
|
||||
|
||||
on_ee do
|
||||
@ -62,14 +62,21 @@ defmodule Plausible.Stats.Base do
|
||||
pageviews:
|
||||
dynamic(
|
||||
[e],
|
||||
fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name)
|
||||
selected_as(
|
||||
fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name),
|
||||
:pageviews
|
||||
)
|
||||
)
|
||||
}
|
||||
end
|
||||
|
||||
defp select_event_metric(:events) do
|
||||
%{
|
||||
events: dynamic([], fragment("toUInt64(round(count(*) * any(_sample_factor)))"))
|
||||
events:
|
||||
dynamic(
|
||||
[],
|
||||
selected_as(fragment("toUInt64(round(count(*) * any(_sample_factor)))"), :events)
|
||||
)
|
||||
}
|
||||
end
|
||||
|
||||
@ -82,7 +89,13 @@ defmodule Plausible.Stats.Base do
|
||||
defp select_event_metric(:visits) do
|
||||
%{
|
||||
visits:
|
||||
dynamic([e], fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.session_id))
|
||||
dynamic(
|
||||
[e],
|
||||
selected_as(
|
||||
fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.session_id),
|
||||
:visits
|
||||
)
|
||||
)
|
||||
}
|
||||
end
|
||||
|
||||
@ -127,10 +140,13 @@ defmodule Plausible.Stats.Base do
|
||||
bounce_rate:
|
||||
dynamic(
|
||||
[],
|
||||
fragment(
|
||||
"toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))",
|
||||
^condition,
|
||||
^condition
|
||||
selected_as(
|
||||
fragment(
|
||||
"toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))",
|
||||
^condition,
|
||||
^condition
|
||||
),
|
||||
:bounce_rate
|
||||
)
|
||||
),
|
||||
__internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))
|
||||
@ -139,7 +155,14 @@ defmodule Plausible.Stats.Base do
|
||||
|
||||
defp select_session_metric(:visits, _query) do
|
||||
%{
|
||||
visits: dynamic([s], fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign))
|
||||
visits:
|
||||
dynamic(
|
||||
[s],
|
||||
selected_as(
|
||||
fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign),
|
||||
:visits
|
||||
)
|
||||
)
|
||||
}
|
||||
end
|
||||
|
||||
@ -148,7 +171,10 @@ defmodule Plausible.Stats.Base do
|
||||
pageviews:
|
||||
dynamic(
|
||||
[s],
|
||||
fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews)
|
||||
selected_as(
|
||||
fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews),
|
||||
:pageviews
|
||||
)
|
||||
)
|
||||
}
|
||||
end
|
||||
@ -158,7 +184,10 @@ defmodule Plausible.Stats.Base do
|
||||
events:
|
||||
dynamic(
|
||||
[s],
|
||||
fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events)
|
||||
selected_as(
|
||||
fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events),
|
||||
:events
|
||||
)
|
||||
)
|
||||
}
|
||||
end
|
||||
@ -179,7 +208,13 @@ defmodule Plausible.Stats.Base do
|
||||
defp select_session_metric(:visit_duration, _query) do
|
||||
%{
|
||||
visit_duration:
|
||||
dynamic([], fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))")),
|
||||
dynamic(
|
||||
[],
|
||||
selected_as(
|
||||
fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))"),
|
||||
:visit_duration
|
||||
)
|
||||
),
|
||||
__internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))
|
||||
}
|
||||
end
|
||||
@ -189,7 +224,15 @@ defmodule Plausible.Stats.Base do
|
||||
views_per_visit:
|
||||
dynamic(
|
||||
[s],
|
||||
fragment("ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)", s.sign, s.pageviews, s.sign)
|
||||
selected_as(
|
||||
fragment(
|
||||
"ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)",
|
||||
s.sign,
|
||||
s.pageviews,
|
||||
s.sign
|
||||
),
|
||||
:views_per_visit
|
||||
)
|
||||
),
|
||||
__internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))
|
||||
}
|
||||
@ -328,11 +371,14 @@ defmodule Plausible.Stats.Base do
|
||||
)
|
||||
|> select_merge(%{
|
||||
percentage:
|
||||
fragment(
|
||||
"if(? > 0, round(? / ? * 100, 1), null)",
|
||||
selected_as(:__total_visitors),
|
||||
selected_as(:visitors),
|
||||
selected_as(:__total_visitors)
|
||||
selected_as(
|
||||
fragment(
|
||||
"if(? > 0, round(? / ? * 100, 1), null)",
|
||||
selected_as(:__total_visitors),
|
||||
selected_as(:visitors),
|
||||
selected_as(:__total_visitors)
|
||||
),
|
||||
:percentage
|
||||
)
|
||||
})
|
||||
else
|
||||
@ -357,11 +403,14 @@ defmodule Plausible.Stats.Base do
|
||||
)
|
||||
|> select_merge([e], %{
|
||||
conversion_rate:
|
||||
fragment(
|
||||
"if(? > 0, round(? / ? * 100, 1), 0)",
|
||||
selected_as(:__total_visitors),
|
||||
e.visitors,
|
||||
selected_as(:__total_visitors)
|
||||
selected_as(
|
||||
fragment(
|
||||
"if(? > 0, round(? / ? * 100, 1), 0)",
|
||||
selected_as(:__total_visitors),
|
||||
e.visitors,
|
||||
selected_as(:__total_visitors)
|
||||
),
|
||||
:conversion_rate
|
||||
)
|
||||
})
|
||||
else
|
||||
|
@ -276,8 +276,8 @@ defmodule Plausible.Stats.Breakdown do
|
||||
defp breakdown_table(%Query{dimensions: ["visit:exit_page"]}, _metrics), do: :session
|
||||
defp breakdown_table(%Query{dimensions: ["visit:exit_page_hostname"]}, _metrics), do: :session
|
||||
|
||||
defp breakdown_table(%Query{dimensions: [dimension]} = query, metrics) do
|
||||
{_, session_metrics, _} = TableDecider.partition_metrics(metrics, query, dimension)
|
||||
defp breakdown_table(%Query{dimensions: [_dimension]} = query, metrics) do
|
||||
{_, session_metrics, _} = TableDecider.partition_metrics(metrics, query)
|
||||
|
||||
if not Enum.empty?(session_metrics) do
|
||||
:session
|
||||
|
@ -87,6 +87,6 @@ defmodule Plausible.Stats.Filters do
|
||||
property
|
||||
|> String.split(":")
|
||||
|> List.last()
|
||||
|> String.to_existing_atom()
|
||||
|> String.to_atom()
|
||||
end
|
||||
end
|
||||
|
@ -1,22 +1,30 @@
|
||||
defmodule Plausible.Stats.Filters.QueryParser do
|
||||
@moduledoc false
|
||||
|
||||
alias Plausible.Stats.TableDecider
|
||||
alias Plausible.Stats.Filters
|
||||
alias Plausible.Stats.Query
|
||||
|
||||
def parse(params) when is_map(params) do
|
||||
def parse(site, params) when is_map(params) do
|
||||
with {:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])),
|
||||
{:ok, filters} <- parse_filters(Map.get(params, "filters", [])),
|
||||
{:ok, date_range} <- parse_date_range(Map.get(params, "date_range")),
|
||||
{:ok, date_range} <- parse_date_range(site, Map.get(params, "date_range")),
|
||||
{:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])),
|
||||
{:ok, order_by} <- parse_order_by(Map.get(params, "order_by")),
|
||||
{:ok, include} <- parse_include(Map.get(params, "include", %{})),
|
||||
query = %{
|
||||
metrics: metrics,
|
||||
filters: filters,
|
||||
date_range: date_range,
|
||||
dimensions: dimensions,
|
||||
order_by: order_by
|
||||
order_by: order_by,
|
||||
timezone: site.timezone,
|
||||
imported_data_requested: Map.get(include, :imports, false)
|
||||
},
|
||||
:ok <- validate_order_by(query) do
|
||||
:ok <- validate_order_by(query),
|
||||
:ok <- validate_goal_filters(site, query),
|
||||
:ok <- validate_custom_props_access(site, query),
|
||||
:ok <- validate_metrics(query) do
|
||||
{:ok, query}
|
||||
end
|
||||
end
|
||||
@ -87,16 +95,26 @@ defmodule Plausible.Stats.Filters.QueryParser do
|
||||
|
||||
defp parse_clauses_list(filter), do: {:error, "Invalid filter '#{inspect(filter)}'"}
|
||||
|
||||
defp parse_date_range("day"), do: {:ok, "day"}
|
||||
defp parse_date_range("7d"), do: {:ok, "7d"}
|
||||
defp parse_date_range("30d"), do: {:ok, "30d"}
|
||||
defp parse_date_range("month"), do: {:ok, "month"}
|
||||
defp parse_date_range("6mo"), do: {:ok, "6mo"}
|
||||
defp parse_date_range("12mo"), do: {:ok, "6mo"}
|
||||
defp parse_date_range("year"), do: {:ok, "year"}
|
||||
defp parse_date_range("all"), do: {:ok, "all"}
|
||||
defp parse_date_range(site, "day") do
|
||||
today = DateTime.now!(site.timezone) |> DateTime.to_date()
|
||||
{:ok, Date.range(today, today)}
|
||||
end
|
||||
|
||||
defp parse_date_range([from_date_string, to_date_string])
|
||||
defp parse_date_range(_site, "7d"), do: {:ok, "7d"}
|
||||
defp parse_date_range(_site, "30d"), do: {:ok, "30d"}
|
||||
defp parse_date_range(_site, "month"), do: {:ok, "month"}
|
||||
defp parse_date_range(_site, "6mo"), do: {:ok, "6mo"}
|
||||
defp parse_date_range(_site, "12mo"), do: {:ok, "12mo"}
|
||||
defp parse_date_range(_site, "year"), do: {:ok, "year"}
|
||||
|
||||
defp parse_date_range(site, "all") do
|
||||
today = DateTime.now!(site.timezone) |> DateTime.to_date()
|
||||
start_date = Plausible.Sites.stats_start_date(site) || today
|
||||
|
||||
{:ok, Date.range(start_date, today)}
|
||||
end
|
||||
|
||||
defp parse_date_range(_site, [from_date_string, to_date_string])
|
||||
when is_bitstring(from_date_string) and is_bitstring(to_date_string) do
|
||||
with {:ok, from_date} <- Date.from_iso8601(from_date_string),
|
||||
{:ok, to_date} <- Date.from_iso8601(to_date_string) do
|
||||
@ -106,13 +124,13 @@ defmodule Plausible.Stats.Filters.QueryParser do
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_date_range(unknown), do: {:error, "Invalid date range '#{inspect(unknown)}'"}
|
||||
defp parse_date_range(_site, unknown), do: {:error, "Invalid date range '#{inspect(unknown)}'"}
|
||||
|
||||
defp parse_dimensions(dimensions) when is_list(dimensions) do
|
||||
if length(dimensions) == length(Enum.uniq(dimensions)) do
|
||||
parse_list(
|
||||
dimensions,
|
||||
&parse_filter_key_string(&1, "Invalid dimensions '#{inspect(dimensions)}'")
|
||||
&parse_dimension_entry(&1, "Invalid dimensions '#{inspect(dimensions)}'")
|
||||
)
|
||||
else
|
||||
{:error, "Some dimensions are listed multiple times"}
|
||||
@ -121,36 +139,67 @@ defmodule Plausible.Stats.Filters.QueryParser do
|
||||
|
||||
defp parse_dimensions(dimensions), do: {:error, "Invalid dimensions '#{inspect(dimensions)}'"}
|
||||
|
||||
def parse_order_by(order_by) when is_list(order_by) do
|
||||
defp parse_order_by(order_by) when is_list(order_by) do
|
||||
parse_list(order_by, &parse_order_by_entry/1)
|
||||
end
|
||||
|
||||
def parse_order_by(nil), do: {:ok, nil}
|
||||
def parse_order_by(order_by), do: {:error, "Invalid order_by '#{inspect(order_by)}'"}
|
||||
defp parse_order_by(nil), do: {:ok, nil}
|
||||
defp parse_order_by(order_by), do: {:error, "Invalid order_by '#{inspect(order_by)}'"}
|
||||
|
||||
def parse_order_by_entry(entry) do
|
||||
with {:ok, metric_or_dimension} <- parse_metric_or_dimension(entry),
|
||||
defp parse_order_by_entry(entry) do
|
||||
with {:ok, value} <- parse_metric_or_dimension(entry),
|
||||
{:ok, order_direction} <- parse_order_direction(entry) do
|
||||
{:ok, {metric_or_dimension, order_direction}}
|
||||
{:ok, {value, order_direction}}
|
||||
end
|
||||
end
|
||||
|
||||
def parse_metric_or_dimension([metric_or_dimension, _] = entry) do
|
||||
case {parse_metric(metric_or_dimension), parse_filter_key_string(metric_or_dimension)} do
|
||||
{{:ok, metric}, _} -> {:ok, metric}
|
||||
{_, {:ok, dimension}} -> {:ok, dimension}
|
||||
defp parse_dimension_entry(key, error_message) do
|
||||
case {
|
||||
parse_time(key),
|
||||
parse_filter_key_string(key)
|
||||
} do
|
||||
{{:ok, time}, _} -> {:ok, time}
|
||||
{_, {:ok, filter_key}} -> {:ok, filter_key}
|
||||
_ -> {:error, error_message}
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_metric_or_dimension([value, _] = entry) do
|
||||
case {
|
||||
parse_time(value),
|
||||
parse_metric(value),
|
||||
parse_filter_key_string(value)
|
||||
} do
|
||||
{{:ok, time}, _, _} -> {:ok, time}
|
||||
{_, {:ok, metric}, _} -> {:ok, metric}
|
||||
{_, _, {:ok, dimension}} -> {:ok, dimension}
|
||||
_ -> {:error, "Invalid order_by entry '#{inspect(entry)}'"}
|
||||
end
|
||||
end
|
||||
|
||||
def parse_order_direction([_, "asc"]), do: {:ok, :asc}
|
||||
def parse_order_direction([_, "desc"]), do: {:ok, :desc}
|
||||
def parse_order_direction(entry), do: {:error, "Invalid order_by entry '#{inspect(entry)}'"}
|
||||
defp parse_time("time"), do: {:ok, "time"}
|
||||
defp parse_time("time:day"), do: {:ok, "time:day"}
|
||||
defp parse_time("time:week"), do: {:ok, "time:week"}
|
||||
defp parse_time("time:month"), do: {:ok, "time:month"}
|
||||
defp parse_time("time:year"), do: {:ok, "time:year"}
|
||||
defp parse_time(_), do: :error
|
||||
|
||||
defp parse_order_direction([_, "asc"]), do: {:ok, :asc}
|
||||
defp parse_order_direction([_, "desc"]), do: {:ok, :desc}
|
||||
defp parse_order_direction(entry), do: {:error, "Invalid order_by entry '#{inspect(entry)}'"}
|
||||
|
||||
defp parse_include(%{"imports" => value}) when is_boolean(value), do: {:ok, %{imports: value}}
|
||||
defp parse_include(%{}), do: {:ok, %{}}
|
||||
defp parse_include(include), do: {:error, "Invalid include passed '#{inspect(include)}'"}
|
||||
|
||||
defp parse_filter_key_string(filter_key, error_message \\ "") do
|
||||
case filter_key do
|
||||
"event:props:" <> _property_name ->
|
||||
{:ok, filter_key}
|
||||
"event:props:" <> property_name ->
|
||||
if String.length(property_name) > 0 do
|
||||
{:ok, filter_key}
|
||||
else
|
||||
{:error, error_message}
|
||||
end
|
||||
|
||||
"event:" <> key ->
|
||||
if key in Filters.event_props() do
|
||||
@ -171,7 +220,7 @@ defmodule Plausible.Stats.Filters.QueryParser do
|
||||
end
|
||||
end
|
||||
|
||||
def validate_order_by(query) do
|
||||
defp validate_order_by(query) do
|
||||
if query.order_by do
|
||||
valid_values = query.metrics ++ query.dimensions
|
||||
|
||||
@ -193,6 +242,92 @@ defmodule Plausible.Stats.Filters.QueryParser do
|
||||
end
|
||||
end
|
||||
|
||||
defp validate_goal_filters(site, query) do
|
||||
goal_filter_clauses =
|
||||
Enum.flat_map(query.filters, fn
|
||||
[_operation, "event:goal", clauses] -> clauses
|
||||
_ -> []
|
||||
end)
|
||||
|
||||
if length(goal_filter_clauses) > 0 do
|
||||
configured_goals =
|
||||
Plausible.Goals.for_site(site)
|
||||
|> Enum.map(fn
|
||||
%{page_path: path} when is_binary(path) -> {:page, path}
|
||||
%{event_name: event_name} -> {:event, event_name}
|
||||
end)
|
||||
|
||||
validate_list(goal_filter_clauses, &validate_goal_filter(&1, configured_goals))
|
||||
else
|
||||
:ok
|
||||
end
|
||||
end
|
||||
|
||||
defp validate_goal_filter(clause, configured_goals) do
|
||||
if Enum.member?(configured_goals, clause) do
|
||||
:ok
|
||||
else
|
||||
{:error,
|
||||
"The goal `#{Filters.Utils.unwrap_goal_value(clause)}` is not configured for this site. Find out how to configure goals here: https://plausible.io/docs/stats-api#filtering-by-goals"}
|
||||
end
|
||||
end
|
||||
|
||||
defp validate_custom_props_access(site, query) do
|
||||
allowed_props = Plausible.Props.allowed_for(site, bypass_setup?: true)
|
||||
|
||||
validate_custom_props_access(site, query, allowed_props)
|
||||
end
|
||||
|
||||
defp validate_custom_props_access(_site, _query, :all), do: :ok
|
||||
|
||||
defp validate_custom_props_access(_site, query, allowed_props) do
|
||||
valid? =
|
||||
query.filters
|
||||
|> Enum.map(fn [_operation, filter_key | _rest] -> filter_key end)
|
||||
|> Enum.concat(query.dimensions)
|
||||
|> Enum.all?(fn
|
||||
"event:props:" <> prop -> prop in allowed_props
|
||||
_ -> true
|
||||
end)
|
||||
|
||||
if valid? do
|
||||
:ok
|
||||
else
|
||||
{:error, "The owner of this site does not have access to the custom properties feature"}
|
||||
end
|
||||
end
|
||||
|
||||
defp validate_metrics(query) do
|
||||
validate_list(query.metrics, &validate_metric(&1, query))
|
||||
|
||||
with :ok <- validate_list(query.metrics, &validate_metric(&1, query)) do
|
||||
validate_no_metrics_filters_conflict(query)
|
||||
end
|
||||
end
|
||||
|
||||
defp validate_metric(:conversion_rate = metric, query) do
|
||||
if Enum.member?(query.dimensions, "event:goal") or
|
||||
not is_nil(Query.get_filter(query, "event:goal")) do
|
||||
:ok
|
||||
else
|
||||
{:error, "Metric `#{metric}` can only be queried with event:goal filters or dimensions"}
|
||||
end
|
||||
end
|
||||
|
||||
defp validate_metric(_, _), do: :ok
|
||||
|
||||
defp validate_no_metrics_filters_conflict(query) do
|
||||
{_event_metrics, sessions_metrics, _other_metrics} =
|
||||
TableDecider.partition_metrics(query.metrics, query)
|
||||
|
||||
if Enum.empty?(sessions_metrics) or not TableDecider.event_filters?(query) do
|
||||
:ok
|
||||
else
|
||||
{:error,
|
||||
"Session metric(s) `#{sessions_metrics |> Enum.join(", ")}` cannot be queried along with event filters or dimensions"}
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_list(list, parser_function) do
|
||||
Enum.reduce_while(list, {:ok, []}, fn value, {:ok, results} ->
|
||||
case parser_function.(value) do
|
||||
@ -201,4 +336,13 @@ defmodule Plausible.Stats.Filters.QueryParser do
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp validate_list(list, parser_function) do
|
||||
Enum.reduce_while(list, :ok, fn value, :ok ->
|
||||
case parser_function.(value) do
|
||||
:ok -> {:cont, :ok}
|
||||
{:error, _} = error -> {:halt, error}
|
||||
end
|
||||
end)
|
||||
end
|
||||
end
|
||||
|
@ -274,18 +274,18 @@ defmodule Plausible.Stats.Imported do
|
||||
site
|
||||
|> Imported.Base.query_imported(query)
|
||||
|> where([i], i.visitors > 0)
|
||||
|> group_imported_by(dim)
|
||||
|> group_imported_by(dim, query)
|
||||
|> select_imported_metrics(metrics)
|
||||
|
||||
join_on =
|
||||
case dim do
|
||||
_ when dim in [:url, :path] ->
|
||||
_ when dim in [:url, :path] and not query.v2 ->
|
||||
dynamic([s, i], s.breakdown_prop_value == i.breakdown_prop_value)
|
||||
|
||||
:os_version ->
|
||||
:os_version when not query.v2 ->
|
||||
dynamic([s, i], s.os == i.os and s.os_version == i.os_version)
|
||||
|
||||
:browser_version ->
|
||||
:browser_version when not query.v2 ->
|
||||
dynamic([s, i], s.browser == i.browser and s.browser_version == i.browser_version)
|
||||
|
||||
dim ->
|
||||
@ -297,7 +297,7 @@ defmodule Plausible.Stats.Imported do
|
||||
on: ^join_on,
|
||||
select: %{}
|
||||
)
|
||||
|> select_joined_dimension(dim)
|
||||
|> select_joined_dimension(dim, query)
|
||||
|> select_joined_metrics(metrics)
|
||||
|> apply_order_by(metrics)
|
||||
end
|
||||
@ -344,7 +344,7 @@ defmodule Plausible.Stats.Imported do
|
||||
on: s.name == i.name,
|
||||
select: %{}
|
||||
)
|
||||
|> select_joined_dimension(:name)
|
||||
|> select_joined_dimension(:name, query)
|
||||
|> select_joined_metrics(metrics)
|
||||
else
|
||||
q
|
||||
@ -551,7 +551,7 @@ defmodule Plausible.Stats.Imported do
|
||||
|> select_imported_metrics(rest)
|
||||
end
|
||||
|
||||
defp group_imported_by(q, dim) when dim in [:source, :referrer] do
|
||||
defp group_imported_by(q, dim, _query) when dim in [:source, :referrer] do
|
||||
q
|
||||
|> group_by([i], field(i, ^dim))
|
||||
|> select_merge([i], %{
|
||||
@ -559,7 +559,7 @@ defmodule Plausible.Stats.Imported do
|
||||
})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, dim)
|
||||
defp group_imported_by(q, dim, _query)
|
||||
when dim in [:utm_source, :utm_medium, :utm_campaign, :utm_term, :utm_content] do
|
||||
q
|
||||
|> group_by([i], field(i, ^dim))
|
||||
@ -567,34 +567,34 @@ defmodule Plausible.Stats.Imported do
|
||||
|> select_merge([i], %{^dim => field(i, ^dim)})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, :page) do
|
||||
defp group_imported_by(q, :page, _query) do
|
||||
q
|
||||
|> group_by([i], i.page)
|
||||
|> select_merge([i], %{page: i.page, time_on_page: sum(i.time_on_page)})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, :country) do
|
||||
defp group_imported_by(q, :country, _query) do
|
||||
q
|
||||
|> group_by([i], i.country)
|
||||
|> where([i], i.country != "ZZ")
|
||||
|> select_merge([i], %{country: i.country})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, :region) do
|
||||
defp group_imported_by(q, :region, _query) do
|
||||
q
|
||||
|> group_by([i], i.region)
|
||||
|> where([i], i.region != "")
|
||||
|> select_merge([i], %{region: i.region})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, :city) do
|
||||
defp group_imported_by(q, :city, _query) do
|
||||
q
|
||||
|> group_by([i], i.city)
|
||||
|> where([i], i.city != 0 and not is_nil(i.city))
|
||||
|> select_merge([i], %{city: i.city})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, dim) when dim in [:device, :browser] do
|
||||
defp group_imported_by(q, dim, _query) when dim in [:device, :browser] do
|
||||
q
|
||||
|> group_by([i], field(i, ^dim))
|
||||
|> select_merge([i], %{
|
||||
@ -602,7 +602,7 @@ defmodule Plausible.Stats.Imported do
|
||||
})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, :browser_version) do
|
||||
defp group_imported_by(q, :browser_version, _query) do
|
||||
q
|
||||
|> group_by([i], [i.browser, i.browser_version])
|
||||
|> select_merge([i], %{
|
||||
@ -617,7 +617,7 @@ defmodule Plausible.Stats.Imported do
|
||||
})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, :os) do
|
||||
defp group_imported_by(q, :os, _query) do
|
||||
q
|
||||
|> group_by([i], i.operating_system)
|
||||
|> select_merge([i], %{
|
||||
@ -625,7 +625,7 @@ defmodule Plausible.Stats.Imported do
|
||||
})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, :os_version) do
|
||||
defp group_imported_by(q, :os_version, _query) do
|
||||
q
|
||||
|> group_by([i], [i.operating_system, i.operating_system_version])
|
||||
|> select_merge([i], %{
|
||||
@ -640,19 +640,27 @@ defmodule Plausible.Stats.Imported do
|
||||
})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, dim) when dim in [:entry_page, :exit_page] do
|
||||
defp group_imported_by(q, dim, _query) when dim in [:entry_page, :exit_page] do
|
||||
q
|
||||
|> group_by([i], field(i, ^dim))
|
||||
|> select_merge([i], %{^dim => field(i, ^dim)})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, :name) do
|
||||
defp group_imported_by(q, :name, _query) do
|
||||
q
|
||||
|> group_by([i], i.name)
|
||||
|> select_merge([i], %{name: i.name})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, :url) do
|
||||
defp group_imported_by(q, :url, query) when query.v2 do
|
||||
q
|
||||
|> group_by([i], i.link_url)
|
||||
|> select_merge([i], %{
|
||||
url: fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)
|
||||
})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, :url, _query) do
|
||||
q
|
||||
|> group_by([i], i.link_url)
|
||||
|> select_merge([i], %{
|
||||
@ -660,7 +668,15 @@ defmodule Plausible.Stats.Imported do
|
||||
})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, :path) do
|
||||
defp group_imported_by(q, :path, query) when query.v2 do
|
||||
q
|
||||
|> group_by([i], i.path)
|
||||
|> select_merge([i], %{
|
||||
path: fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)
|
||||
})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, :path, _query) do
|
||||
q
|
||||
|> group_by([i], i.path)
|
||||
|> select_merge([i], %{
|
||||
@ -668,20 +684,20 @@ defmodule Plausible.Stats.Imported do
|
||||
})
|
||||
end
|
||||
|
||||
defp select_joined_dimension(q, :city) do
|
||||
defp select_joined_dimension(q, :city, _query) do
|
||||
select_merge(q, [s, i], %{
|
||||
city: fragment("greatest(?,?)", i.city, s.city)
|
||||
})
|
||||
end
|
||||
|
||||
defp select_joined_dimension(q, :os_version) do
|
||||
defp select_joined_dimension(q, :os_version, query) when not query.v2 do
|
||||
select_merge(q, [s, i], %{
|
||||
os: fragment("if(empty(?), ?, ?)", s.os, i.os, s.os),
|
||||
os_version: fragment("if(empty(?), ?, ?)", s.os_version, i.os_version, s.os_version)
|
||||
})
|
||||
end
|
||||
|
||||
defp select_joined_dimension(q, :browser_version) do
|
||||
defp select_joined_dimension(q, :browser_version, query) when not query.v2 do
|
||||
select_merge(q, [s, i], %{
|
||||
browser: fragment("if(empty(?), ?, ?)", s.browser, i.browser, s.browser),
|
||||
browser_version:
|
||||
@ -689,7 +705,7 @@ defmodule Plausible.Stats.Imported do
|
||||
})
|
||||
end
|
||||
|
||||
defp select_joined_dimension(q, dim) when dim in [:url, :path] do
|
||||
defp select_joined_dimension(q, dim, query) when dim in [:url, :path] and not query.v2 do
|
||||
select_merge(q, [s, i], %{
|
||||
breakdown_prop_value:
|
||||
fragment(
|
||||
@ -701,7 +717,7 @@ defmodule Plausible.Stats.Imported do
|
||||
})
|
||||
end
|
||||
|
||||
defp select_joined_dimension(q, dim) do
|
||||
defp select_joined_dimension(q, dim, _query) do
|
||||
select_merge(q, [s, i], %{
|
||||
^dim => fragment("if(empty(?), ?, ?)", field(s, ^dim), field(i, ^dim), field(s, ^dim))
|
||||
})
|
||||
@ -716,7 +732,7 @@ defmodule Plausible.Stats.Imported do
|
||||
|
||||
defp select_joined_metrics(q, [:visits | rest]) do
|
||||
q
|
||||
|> select_merge([s, i], %{visits: s.visits + i.visits})
|
||||
|> select_merge([s, i], %{visits: selected_as(s.visits + i.visits, :visits)})
|
||||
|> select_joined_metrics(rest)
|
||||
end
|
||||
|
||||
@ -728,13 +744,13 @@ defmodule Plausible.Stats.Imported do
|
||||
|
||||
defp select_joined_metrics(q, [:events | rest]) do
|
||||
q
|
||||
|> select_merge([s, i], %{events: s.events + i.events})
|
||||
|> select_merge([s, i], %{events: selected_as(s.events + i.events, :events)})
|
||||
|> select_joined_metrics(rest)
|
||||
end
|
||||
|
||||
defp select_joined_metrics(q, [:pageviews | rest]) do
|
||||
q
|
||||
|> select_merge([s, i], %{pageviews: s.pageviews + i.pageviews})
|
||||
|> select_merge([s, i], %{pageviews: selected_as(s.pageviews + i.pageviews, :pageviews)})
|
||||
|> select_joined_metrics(rest)
|
||||
end
|
||||
|
||||
|
@ -21,6 +21,8 @@ defmodule Plausible.Stats.Metrics do
|
||||
|
||||
@metric_mappings Enum.into(@all_metrics, %{}, fn metric -> {to_string(metric), metric} end)
|
||||
|
||||
def metric?(value), do: Enum.member?(@all_metrics, value)
|
||||
|
||||
def from_string!(str) do
|
||||
Map.fetch!(@metric_mappings, str)
|
||||
end
|
||||
|
@ -13,7 +13,11 @@ defmodule Plausible.Stats.Query do
|
||||
now: nil,
|
||||
experimental_session_count?: false,
|
||||
experimental_reduced_joins?: false,
|
||||
latest_import_end_date: nil
|
||||
latest_import_end_date: nil,
|
||||
metrics: [],
|
||||
order_by: [],
|
||||
timezone: nil,
|
||||
v2: false
|
||||
|
||||
require OpenTelemetry.Tracer, as: Tracer
|
||||
alias Plausible.Stats.{Filters, Interval, Imported}
|
||||
@ -42,6 +46,19 @@ defmodule Plausible.Stats.Query do
|
||||
query
|
||||
end
|
||||
|
||||
def build(site, params) do
|
||||
with {:ok, query_data} <- Filters.QueryParser.parse(site, params) do
|
||||
query =
|
||||
struct!(__MODULE__, Map.to_list(query_data))
|
||||
|> put_imported_opts(site, %{})
|
||||
|> put_experimental_session_count(site, params)
|
||||
|> put_experimental_reduced_joins(site, params)
|
||||
|> struct!(v2: true)
|
||||
|
||||
{:ok, query}
|
||||
end
|
||||
end
|
||||
|
||||
defp put_experimental_session_count(query, site, params) do
|
||||
if Map.has_key?(params, "experimental_session_count") do
|
||||
struct!(query,
|
||||
|
28
lib/plausible/stats/query_optimizer.ex
Normal file
28
lib/plausible/stats/query_optimizer.ex
Normal file
@ -0,0 +1,28 @@
|
||||
defmodule Plausible.Stats.QueryOptimizer do
|
||||
@moduledoc false
|
||||
|
||||
alias Plausible.Stats.Query
|
||||
|
||||
def optimize(query) do
|
||||
Enum.reduce(pipeline(), query, fn step, acc -> step.(acc) end)
|
||||
end
|
||||
|
||||
defp pipeline() do
|
||||
[
|
||||
&add_missing_order_by/1,
|
||||
&update_group_by_time/1
|
||||
]
|
||||
end
|
||||
|
||||
defp add_missing_order_by(%Query{order_by: nil} = query) do
|
||||
%Query{query | order_by: [{hd(query.metrics), :desc}]}
|
||||
end
|
||||
|
||||
defp add_missing_order_by(query), do: query
|
||||
|
||||
defp update_group_by_time(%Query{dimensions: ["time" | rest]} = query) do
|
||||
%Query{query | dimensions: ["time:month" | rest]}
|
||||
end
|
||||
|
||||
defp update_group_by_time(query), do: query
|
||||
end
|
52
lib/plausible/stats/query_result.ex
Normal file
52
lib/plausible/stats/query_result.ex
Normal file
@ -0,0 +1,52 @@
|
||||
defmodule Plausible.Stats.QueryResult do
|
||||
@moduledoc false
|
||||
|
||||
alias Plausible.Stats.SQL.QueryBuilder
|
||||
alias Plausible.Stats.Filters
|
||||
alias Plausible.Stats.Query
|
||||
|
||||
@derive Jason.Encoder
|
||||
defstruct results: [],
|
||||
query: nil,
|
||||
meta: %{}
|
||||
|
||||
def from(results, query) do
|
||||
results_list =
|
||||
results
|
||||
|> Enum.map(fn entry ->
|
||||
%{
|
||||
dimensions: Enum.map(query.dimensions, &Map.get(entry, QueryBuilder.shortname(&1))),
|
||||
metrics: Enum.map(query.metrics, &Map.get(entry, &1))
|
||||
}
|
||||
end)
|
||||
|
||||
struct!(
|
||||
__MODULE__,
|
||||
results: results_list,
|
||||
query: %{
|
||||
metrics: query.metrics,
|
||||
date_range: [query.date_range.first, query.date_range.last],
|
||||
filters: query.filters |> Enum.map(&serializable_filter/1),
|
||||
dimensions: query.dimensions,
|
||||
order_by: query.order_by |> Enum.map(&Tuple.to_list/1)
|
||||
},
|
||||
meta: meta(query)
|
||||
)
|
||||
end
|
||||
|
||||
defp meta(%Query{skip_imported_reason: :unsupported_query}) do
|
||||
%{
|
||||
warning:
|
||||
"Imported stats are not included in the results because query parameters are not supported. " <>
|
||||
"For more information, see: https://plausible.io/docs/stats-api#filtering-imported-stats"
|
||||
}
|
||||
end
|
||||
|
||||
defp meta(_), do: %{}
|
||||
|
||||
defp serializable_filter([operation, "event:goal", clauses]) do
|
||||
[operation, "event:goal", Enum.map(clauses, &Filters.Utils.unwrap_goal_value/1)]
|
||||
end
|
||||
|
||||
defp serializable_filter(filter), do: filter
|
||||
end
|
89
lib/plausible/stats/sql/expression.ex
Normal file
89
lib/plausible/stats/sql/expression.ex
Normal file
@ -0,0 +1,89 @@
|
||||
defmodule Plausible.Stats.SQL.Expression do
|
||||
@moduledoc false
|
||||
|
||||
import Ecto.Query
|
||||
|
||||
use Plausible.Stats.Fragments
|
||||
|
||||
@no_ref "Direct / None"
|
||||
@not_set "(not set)"
|
||||
|
||||
defmacrop field_or_blank_value(expr, empty_value) do
|
||||
quote do
|
||||
dynamic(
|
||||
[t],
|
||||
fragment("if(empty(?), ?, ?)", unquote(expr), unquote(empty_value), unquote(expr))
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
def dimension("time:hour", query) do
|
||||
dynamic([t], fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone))
|
||||
end
|
||||
|
||||
def dimension("time:day", query) do
|
||||
dynamic([t], fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone))
|
||||
end
|
||||
|
||||
def dimension("time:month", query) do
|
||||
dynamic([t], fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone))
|
||||
end
|
||||
|
||||
def dimension("event:name", _query), do: dynamic([t], t.name)
|
||||
def dimension("event:page", _query), do: dynamic([t], t.pathname)
|
||||
def dimension("event:hostname", _query), do: dynamic([t], t.hostname)
|
||||
|
||||
def dimension("event:props:" <> property_name, _query) do
|
||||
dynamic(
|
||||
[t],
|
||||
fragment(
|
||||
"if(not empty(?), ?, '(none)')",
|
||||
get_by_key(t, :meta, ^property_name),
|
||||
get_by_key(t, :meta, ^property_name)
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
def dimension("visit:entry_page", _query), do: dynamic([t], t.entry_page)
|
||||
def dimension("visit:exit_page", _query), do: dynamic([t], t.exit_page)
|
||||
|
||||
def dimension("visit:utm_medium", _query),
|
||||
do: field_or_blank_value(t.utm_medium, @not_set)
|
||||
|
||||
def dimension("visit:utm_source", _query),
|
||||
do: field_or_blank_value(t.utm_source, @not_set)
|
||||
|
||||
def dimension("visit:utm_campaign", _query),
|
||||
do: field_or_blank_value(t.utm_campaign, @not_set)
|
||||
|
||||
def dimension("visit:utm_content", _query),
|
||||
do: field_or_blank_value(t.utm_content, @not_set)
|
||||
|
||||
def dimension("visit:utm_term", _query),
|
||||
do: field_or_blank_value(t.utm_term, @not_set)
|
||||
|
||||
def dimension("visit:source", _query),
|
||||
do: field_or_blank_value(t.source, @no_ref)
|
||||
|
||||
def dimension("visit:referrer", _query),
|
||||
do: field_or_blank_value(t.referrer, @no_ref)
|
||||
|
||||
def dimension("visit:device", _query),
|
||||
do: field_or_blank_value(t.device, @not_set)
|
||||
|
||||
def dimension("visit:os", _query), do: field_or_blank_value(t.os, @not_set)
|
||||
|
||||
def dimension("visit:os_version", _query),
|
||||
do: field_or_blank_value(t.os_version, @not_set)
|
||||
|
||||
def dimension("visit:browser", _query),
|
||||
do: field_or_blank_value(t.browser, @not_set)
|
||||
|
||||
def dimension("visit:browser_version", _query),
|
||||
do: field_or_blank_value(t.browser_version, @not_set)
|
||||
|
||||
# :TODO: Locations also set extra filters
|
||||
def dimension("visit:country", _query), do: dynamic([t], t.country)
|
||||
def dimension("visit:region", _query), do: dynamic([t], t.region)
|
||||
def dimension("visit:city", _query), do: dynamic([t], t.city)
|
||||
end
|
200
lib/plausible/stats/sql/query_builder.ex
Normal file
200
lib/plausible/stats/sql/query_builder.ex
Normal file
@ -0,0 +1,200 @@
|
||||
defmodule Plausible.Stats.SQL.QueryBuilder do
|
||||
@moduledoc false
|
||||
|
||||
use Plausible
|
||||
|
||||
import Ecto.Query
|
||||
import Plausible.Stats.Imported
|
||||
|
||||
alias Plausible.Stats.{Base, Query, TableDecider, Util, Filters, Metrics}
|
||||
alias Plausible.Stats.SQL.Expression
|
||||
|
||||
def build(query, site) do
|
||||
{event_metrics, sessions_metrics, _other_metrics} =
|
||||
query.metrics
|
||||
|> Util.maybe_add_visitors_metric()
|
||||
|> TableDecider.partition_metrics(query)
|
||||
|
||||
join_query_results(
|
||||
build_events_query(site, query, event_metrics),
|
||||
event_metrics,
|
||||
build_sessions_query(site, query, sessions_metrics),
|
||||
sessions_metrics,
|
||||
query
|
||||
)
|
||||
end
|
||||
|
||||
def shortname(metric) when is_atom(metric), do: metric
|
||||
def shortname(dimension), do: Plausible.Stats.Filters.without_prefix(dimension)
|
||||
|
||||
defp build_events_query(_, _, []), do: nil
|
||||
|
||||
defp build_events_query(site, query, event_metrics) do
|
||||
q =
|
||||
from(
|
||||
e in "events_v2",
|
||||
where: ^Filters.WhereBuilder.build(:events, site, query),
|
||||
select: ^Base.select_event_metrics(event_metrics)
|
||||
)
|
||||
|
||||
on_ee do
|
||||
q = Plausible.Stats.Sampling.add_query_hint(q, query)
|
||||
end
|
||||
|
||||
q
|
||||
|> join_sessions_if_needed(site, query)
|
||||
|> build_group_by(query)
|
||||
|> merge_imported(site, query, event_metrics)
|
||||
|> Base.maybe_add_conversion_rate(site, query, event_metrics)
|
||||
end
|
||||
|
||||
defp join_sessions_if_needed(q, site, query) do
|
||||
if TableDecider.events_join_sessions?(query) do
|
||||
sessions_q =
|
||||
from(
|
||||
s in Base.query_sessions(site, query),
|
||||
select: %{session_id: s.session_id},
|
||||
where: s.sign == 1,
|
||||
group_by: s.session_id
|
||||
)
|
||||
|
||||
from(
|
||||
e in q,
|
||||
join: sq in subquery(sessions_q),
|
||||
on: e.session_id == sq.session_id
|
||||
)
|
||||
else
|
||||
q
|
||||
end
|
||||
end
|
||||
|
||||
def build_sessions_query(_, _, []), do: nil
|
||||
|
||||
def build_sessions_query(site, query, session_metrics) do
|
||||
q =
|
||||
from(
|
||||
e in "sessions_v2",
|
||||
where: ^Filters.WhereBuilder.build(:sessions, site, query),
|
||||
select: ^Base.select_session_metrics(session_metrics, query)
|
||||
)
|
||||
|
||||
on_ee do
|
||||
q = Plausible.Stats.Sampling.add_query_hint(q, query)
|
||||
end
|
||||
|
||||
q
|
||||
|> join_events_if_needed(site, query)
|
||||
|> build_group_by(query)
|
||||
|> merge_imported(site, query, session_metrics)
|
||||
end
|
||||
|
||||
def join_events_if_needed(q, site, query) do
|
||||
if Query.has_event_filters?(query) do
|
||||
events_q =
|
||||
from(e in "events_v2",
|
||||
where: ^Filters.WhereBuilder.build(:events, site, query),
|
||||
select: %{
|
||||
session_id: fragment("DISTINCT ?", e.session_id),
|
||||
_sample_factor: fragment("_sample_factor")
|
||||
}
|
||||
)
|
||||
|
||||
on_ee do
|
||||
events_q = Plausible.Stats.Sampling.add_query_hint(events_q, query)
|
||||
end
|
||||
|
||||
from(s in q,
|
||||
join: e in subquery(events_q),
|
||||
on: s.session_id == e.session_id
|
||||
)
|
||||
else
|
||||
q
|
||||
end
|
||||
end
|
||||
|
||||
defp build_group_by(q, query) do
|
||||
Enum.reduce(query.dimensions, q, fn dimension, q ->
|
||||
q
|
||||
|> select_merge(^%{shortname(dimension) => Expression.dimension(dimension, query)})
|
||||
|> group_by(^Expression.dimension(dimension, query))
|
||||
end)
|
||||
end
|
||||
|
||||
defp build_order_by(q, query, mode) do
|
||||
Enum.reduce(query.order_by, q, &build_order_by(&2, query, &1, mode))
|
||||
end
|
||||
|
||||
def build_order_by(q, query, {metric_or_dimension, order_direction}, :inner) do
|
||||
order_by(
|
||||
q,
|
||||
[t],
|
||||
^{
|
||||
order_direction,
|
||||
if(
|
||||
Metrics.metric?(metric_or_dimension),
|
||||
do: dynamic([], selected_as(^shortname(metric_or_dimension))),
|
||||
else: Expression.dimension(metric_or_dimension, query)
|
||||
)
|
||||
}
|
||||
)
|
||||
end
|
||||
|
||||
def build_order_by(q, _query, {metric_or_dimension, order_direction}, :outer) do
|
||||
order_by(
|
||||
q,
|
||||
[t],
|
||||
^{
|
||||
order_direction,
|
||||
dynamic([], selected_as(^shortname(metric_or_dimension)))
|
||||
}
|
||||
)
|
||||
end
|
||||
|
||||
defmacrop select_join_fields(q, list, table_name) do
|
||||
quote do
|
||||
Enum.reduce(unquote(list), unquote(q), fn metric_or_dimension, q ->
|
||||
select_merge(
|
||||
q,
|
||||
^%{
|
||||
shortname(metric_or_dimension) =>
|
||||
dynamic(
|
||||
[e, s],
|
||||
selected_as(
|
||||
field(unquote(table_name), ^shortname(metric_or_dimension)),
|
||||
^shortname(metric_or_dimension)
|
||||
)
|
||||
)
|
||||
}
|
||||
)
|
||||
end)
|
||||
end
|
||||
end
|
||||
|
||||
defp join_query_results(nil, _, nil, _, _query), do: nil
|
||||
|
||||
defp join_query_results(events_q, _, nil, _, query),
|
||||
do: events_q |> build_order_by(query, :inner)
|
||||
|
||||
defp join_query_results(nil, _, sessions_q, _, query),
|
||||
do: sessions_q |> build_order_by(query, :inner)
|
||||
|
||||
defp join_query_results(events_q, event_metrics, sessions_q, sessions_metrics, query) do
|
||||
join(subquery(events_q), :left, [e], s in subquery(sessions_q),
|
||||
on: ^build_group_by_join(query)
|
||||
)
|
||||
|> select_join_fields(query.dimensions, e)
|
||||
|> select_join_fields(event_metrics, e)
|
||||
|> select_join_fields(List.delete(sessions_metrics, :sample_percent), s)
|
||||
|> build_order_by(query, :outer)
|
||||
end
|
||||
|
||||
defp build_group_by_join(%Query{dimensions: []}), do: true
|
||||
|
||||
defp build_group_by_join(query) do
|
||||
query.dimensions
|
||||
|> Enum.map(fn dim ->
|
||||
dynamic([e, s], field(e, ^shortname(dim)) == field(s, ^shortname(dim)))
|
||||
end)
|
||||
|> Enum.reduce(fn condition, acc -> dynamic([], ^acc and ^condition) end)
|
||||
end
|
||||
end
|
@ -9,10 +9,18 @@ defmodule Plausible.Stats.TableDecider do
|
||||
alias Plausible.Stats.Query
|
||||
|
||||
def events_join_sessions?(query) do
|
||||
Enum.any?(query.filters, &(filters_partitioner(query, &1) == :session))
|
||||
query
|
||||
|> filter_keys()
|
||||
|> Enum.any?(&(filters_partitioner(query, &1) == :session))
|
||||
end
|
||||
|
||||
def partition_metrics(metrics, query, breakdown_property \\ nil) do
|
||||
def event_filters?(query) do
|
||||
query
|
||||
|> filter_keys()
|
||||
|> Enum.any?(&(filters_partitioner(query, &1) == :event))
|
||||
end
|
||||
|
||||
def partition_metrics(metrics, query) do
|
||||
%{
|
||||
event: event_only_metrics,
|
||||
session: session_only_metrics,
|
||||
@ -22,16 +30,10 @@ defmodule Plausible.Stats.TableDecider do
|
||||
} =
|
||||
partition(metrics, query, &metric_partitioner/2)
|
||||
|
||||
# Treat breakdown property as yet another filter
|
||||
query =
|
||||
if breakdown_property do
|
||||
Query.put_filter(query, [:is, breakdown_property, []])
|
||||
else
|
||||
query
|
||||
end
|
||||
|
||||
%{event: event_only_filters, session: session_only_filters} =
|
||||
partition(query.filters, query, &filters_partitioner/2)
|
||||
query
|
||||
|> filter_keys()
|
||||
|> partition(query, &filters_partitioner/2)
|
||||
|
||||
cond do
|
||||
# Only one table needs to be queried
|
||||
@ -55,6 +57,12 @@ defmodule Plausible.Stats.TableDecider do
|
||||
end
|
||||
end
|
||||
|
||||
defp filter_keys(query) do
|
||||
query.filters
|
||||
|> Enum.map(fn [_, filter_key | _rest] -> filter_key end)
|
||||
|> Enum.concat(query.dimensions)
|
||||
end
|
||||
|
||||
defp metric_partitioner(_, :conversion_rate), do: :event
|
||||
defp metric_partitioner(_, :average_revenue), do: :event
|
||||
defp metric_partitioner(_, :total_revenue), do: :event
|
||||
@ -83,16 +91,16 @@ defmodule Plausible.Stats.TableDecider do
|
||||
|
||||
defp metric_partitioner(_, _), do: :either
|
||||
|
||||
defp filters_partitioner(_, [_, "event:" <> _ | _rest]), do: :event
|
||||
defp filters_partitioner(_, [_, "visit:entry_page" | _rest]), do: :session
|
||||
defp filters_partitioner(_, [_, "visit:entry_page_hostname" | _rest]), do: :session
|
||||
defp filters_partitioner(_, [_, "visit:exit_page" | _rest]), do: :session
|
||||
defp filters_partitioner(_, [_, "visit:exit_page_hostname" | _rest]), do: :session
|
||||
defp filters_partitioner(_, "event:" <> _), do: :event
|
||||
defp filters_partitioner(_, "visit:entry_page"), do: :session
|
||||
defp filters_partitioner(_, "visit:entry_page_hostname"), do: :session
|
||||
defp filters_partitioner(_, "visit:exit_page"), do: :session
|
||||
defp filters_partitioner(_, "visit:exit_page_hostname"), do: :session
|
||||
|
||||
defp filters_partitioner(%Query{experimental_reduced_joins?: true}, [_, "visit:" <> _ | _rest]),
|
||||
defp filters_partitioner(%Query{experimental_reduced_joins?: true}, "visit:" <> _),
|
||||
do: :either
|
||||
|
||||
defp filters_partitioner(_, [_, "visit:" <> _ | _rest]),
|
||||
defp filters_partitioner(_, "visit:" <> _),
|
||||
do: :session
|
||||
|
||||
defp filters_partitioner(%Query{experimental_reduced_joins?: false}, {unknown, _}) do
|
||||
|
@ -0,0 +1,23 @@
|
||||
defmodule PlausibleWeb.Api.ExternalQueryApiController do
|
||||
@moduledoc false
|
||||
|
||||
use PlausibleWeb, :controller
|
||||
use Plausible.Repo
|
||||
use PlausibleWeb.Plugs.ErrorHandler
|
||||
alias Plausible.Stats.Query
|
||||
|
||||
def query(conn, params) do
|
||||
site = Repo.preload(conn.assigns.site, :owner)
|
||||
|
||||
case Query.build(site, params) do
|
||||
{:ok, query} ->
|
||||
results = Plausible.Stats.query(site, query)
|
||||
json(conn, results)
|
||||
|
||||
{:error, message} ->
|
||||
conn
|
||||
|> put_status(400)
|
||||
|> json(%{error: message})
|
||||
end
|
||||
end
|
||||
end
|
@ -175,6 +175,12 @@ defmodule PlausibleWeb.Router do
|
||||
get "/timeseries", ExternalStatsController, :timeseries
|
||||
end
|
||||
|
||||
scope "/api/v2", PlausibleWeb.Api do
|
||||
pipe_through [:public_api, PlausibleWeb.AuthorizeStatsApiPlug]
|
||||
|
||||
post "/query", ExternalQueryApiController, :query
|
||||
end
|
||||
|
||||
on_ee do
|
||||
scope "/api/v1/sites", PlausibleWeb.Api do
|
||||
pipe_through [:public_api, PlausibleWeb.AuthorizeSitesApiPlug]
|
||||
|
@ -1,44 +1,50 @@
|
||||
defmodule Plausible.Stats.Filters.QueryParserTest do
|
||||
use ExUnit.Case, async: true
|
||||
use Plausible.DataCase
|
||||
|
||||
alias Plausible.Stats.Filters
|
||||
import Plausible.Stats.Filters.QueryParser
|
||||
|
||||
def check_success(params, expected_result) do
|
||||
assert parse(params) == {:ok, expected_result}
|
||||
setup [:create_user, :create_new_site]
|
||||
|
||||
@date_range Date.range(Timex.today(), Timex.today())
|
||||
|
||||
def check_success(params, site, expected_result) do
|
||||
assert parse(site, params) == {:ok, expected_result}
|
||||
end
|
||||
|
||||
def check_error(params, expected_error_message) do
|
||||
{:error, message} = parse(params)
|
||||
def check_error(params, site, expected_error_message) do
|
||||
{:error, message} = parse(site, params)
|
||||
assert message =~ expected_error_message
|
||||
end
|
||||
|
||||
test "parsing empty map fails" do
|
||||
test "parsing empty map fails", %{site: site} do
|
||||
%{}
|
||||
|> check_error("No valid metrics passed")
|
||||
|> check_error(site, "No valid metrics passed")
|
||||
end
|
||||
|
||||
describe "metrics validation" do
|
||||
test "valid metrics passed" do
|
||||
test "valid metrics passed", %{site: site} do
|
||||
%{"metrics" => ["visitors", "events"], "date_range" => "all"}
|
||||
|> check_success(%{
|
||||
|> check_success(site, %{
|
||||
metrics: [:visitors, :events],
|
||||
date_range: "all",
|
||||
date_range: @date_range,
|
||||
filters: [],
|
||||
dimensions: [],
|
||||
order_by: nil
|
||||
order_by: nil,
|
||||
timezone: site.timezone,
|
||||
imported_data_requested: false
|
||||
})
|
||||
end
|
||||
|
||||
test "invalid metric passed" do
|
||||
test "invalid metric passed", %{site: site} do
|
||||
%{"metrics" => ["visitors", "event:name"], "date_range" => "all"}
|
||||
|> check_error("Unknown metric '\"event:name\"'")
|
||||
|> check_error(site, "Unknown metric '\"event:name\"'")
|
||||
end
|
||||
|
||||
test "fuller list of metrics" do
|
||||
test "fuller list of metrics", %{site: site} do
|
||||
%{
|
||||
"metrics" => [
|
||||
"time_on_page",
|
||||
"conversion_rate",
|
||||
"visitors",
|
||||
"pageviews",
|
||||
"visits",
|
||||
@ -48,10 +54,9 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
|
||||
],
|
||||
"date_range" => "all"
|
||||
}
|
||||
|> check_success(%{
|
||||
|> check_success(site, %{
|
||||
metrics: [
|
||||
:time_on_page,
|
||||
:conversion_rate,
|
||||
:visitors,
|
||||
:pageviews,
|
||||
:visits,
|
||||
@ -59,22 +64,24 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
|
||||
:bounce_rate,
|
||||
:visit_duration
|
||||
],
|
||||
date_range: "all",
|
||||
date_range: @date_range,
|
||||
filters: [],
|
||||
dimensions: [],
|
||||
order_by: nil
|
||||
order_by: nil,
|
||||
timezone: site.timezone,
|
||||
imported_data_requested: false
|
||||
})
|
||||
end
|
||||
|
||||
test "same metric queried multiple times" do
|
||||
test "same metric queried multiple times", %{site: site} do
|
||||
%{"metrics" => ["events", "visitors", "visitors"], "date_range" => "all"}
|
||||
|> check_error(~r/Metrics cannot be queried multiple times/)
|
||||
|> check_error(site, ~r/Metrics cannot be queried multiple times/)
|
||||
end
|
||||
end
|
||||
|
||||
describe "filters validation" do
|
||||
for operation <- [:is, :is_not, :matches, :does_not_match] do
|
||||
test "#{operation} filter" do
|
||||
test "#{operation} filter", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
@ -82,18 +89,20 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
|
||||
[Atom.to_string(unquote(operation)), "event:name", ["foo"]]
|
||||
]
|
||||
}
|
||||
|> check_success(%{
|
||||
|> check_success(site, %{
|
||||
metrics: [:visitors],
|
||||
date_range: "all",
|
||||
date_range: @date_range,
|
||||
filters: [
|
||||
[unquote(operation), "event:name", ["foo"]]
|
||||
],
|
||||
dimensions: [],
|
||||
order_by: nil
|
||||
order_by: nil,
|
||||
timezone: site.timezone,
|
||||
imported_data_requested: false
|
||||
})
|
||||
end
|
||||
|
||||
test "#{operation} filter with invalid clause" do
|
||||
test "#{operation} filter with invalid clause", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
@ -101,11 +110,11 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
|
||||
[Atom.to_string(unquote(operation)), "event:name", "foo"]
|
||||
]
|
||||
}
|
||||
|> check_error(~r/Invalid filter/)
|
||||
|> check_error(site, ~r/Invalid filter/)
|
||||
end
|
||||
end
|
||||
|
||||
test "filtering by invalid operation" do
|
||||
test "filtering by invalid operation", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
@ -113,10 +122,10 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
|
||||
["exists?", "event:name", ["foo"]]
|
||||
]
|
||||
}
|
||||
|> check_error(~r/Unknown operator for filter/)
|
||||
|> check_error(site, ~r/Unknown operator for filter/)
|
||||
end
|
||||
|
||||
test "filtering by custom properties" do
|
||||
test "filtering by custom properties", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
@ -124,20 +133,22 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
|
||||
["is", "event:props:foobar", ["value"]]
|
||||
]
|
||||
}
|
||||
|> check_success(%{
|
||||
|> check_success(site, %{
|
||||
metrics: [:visitors],
|
||||
date_range: "all",
|
||||
date_range: @date_range,
|
||||
filters: [
|
||||
[:is, "event:props:foobar", ["value"]]
|
||||
],
|
||||
dimensions: [],
|
||||
order_by: nil
|
||||
order_by: nil,
|
||||
timezone: site.timezone,
|
||||
imported_data_requested: false
|
||||
})
|
||||
end
|
||||
|
||||
for dimension <- Filters.event_props() do
|
||||
if dimension != "goal" do
|
||||
test "filtering by event:#{dimension} filter" do
|
||||
test "filtering by event:#{dimension} filter", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
@ -145,21 +156,23 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
|
||||
["is", "event:#{unquote(dimension)}", ["foo"]]
|
||||
]
|
||||
}
|
||||
|> check_success(%{
|
||||
|> check_success(site, %{
|
||||
metrics: [:visitors],
|
||||
date_range: "all",
|
||||
date_range: @date_range,
|
||||
filters: [
|
||||
[:is, "event:#{unquote(dimension)}", ["foo"]]
|
||||
],
|
||||
dimensions: [],
|
||||
order_by: nil
|
||||
order_by: nil,
|
||||
timezone: site.timezone,
|
||||
imported_data_requested: false
|
||||
})
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
for dimension <- Filters.visit_props() do
|
||||
test "filtering by visit:#{dimension} filter" do
|
||||
test "filtering by visit:#{dimension} filter", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
@ -167,38 +180,21 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
|
||||
["is", "visit:#{unquote(dimension)}", ["foo"]]
|
||||
]
|
||||
}
|
||||
|> check_success(%{
|
||||
|> check_success(site, %{
|
||||
metrics: [:visitors],
|
||||
date_range: "all",
|
||||
date_range: @date_range,
|
||||
filters: [
|
||||
[:is, "visit:#{unquote(dimension)}", ["foo"]]
|
||||
],
|
||||
dimensions: [],
|
||||
order_by: nil
|
||||
order_by: nil,
|
||||
timezone: site.timezone,
|
||||
imported_data_requested: false
|
||||
})
|
||||
end
|
||||
end
|
||||
|
||||
test "filtering by event:goal" do
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
"filters" => [
|
||||
["is", "event:goal", ["Signup", "Visit /thank-you"]]
|
||||
]
|
||||
}
|
||||
|> check_success(%{
|
||||
metrics: [:visitors],
|
||||
date_range: "all",
|
||||
filters: [
|
||||
[:is, "event:goal", [{:event, "Signup"}, {:page, "/thank-you"}]]
|
||||
],
|
||||
dimensions: [],
|
||||
order_by: nil
|
||||
})
|
||||
end
|
||||
|
||||
test "invalid event filter" do
|
||||
test "invalid event filter", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
@ -206,10 +202,10 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
|
||||
["is", "event:device", ["foo"]]
|
||||
]
|
||||
}
|
||||
|> check_error(~r/Invalid filter /)
|
||||
|> check_error(site, ~r/Invalid filter /)
|
||||
end
|
||||
|
||||
test "invalid visit filter" do
|
||||
test "invalid visit filter", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
@ -217,16 +213,92 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
|
||||
["is", "visit:name", ["foo"]]
|
||||
]
|
||||
}
|
||||
|> check_error(~r/Invalid filter /)
|
||||
|> check_error(site, ~r/Invalid filter /)
|
||||
end
|
||||
|
||||
test "invalid filter" do
|
||||
test "invalid filter", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
"filters" => "foobar"
|
||||
}
|
||||
|> check_error(~r/Invalid filters passed/)
|
||||
|> check_error(site, ~r/Invalid filters passed/)
|
||||
end
|
||||
end
|
||||
|
||||
describe "include validation" do
|
||||
test "setting include.imports", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
"include" => %{"imports" => true}
|
||||
}
|
||||
|> check_success(site, %{
|
||||
metrics: [:visitors],
|
||||
date_range: @date_range,
|
||||
filters: [],
|
||||
dimensions: [],
|
||||
order_by: nil,
|
||||
timezone: site.timezone,
|
||||
imported_data_requested: true
|
||||
})
|
||||
end
|
||||
|
||||
test "setting invalid imports value", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
"include" => "foobar"
|
||||
}
|
||||
|> check_error(site, ~r/Invalid include passed/)
|
||||
end
|
||||
end
|
||||
|
||||
describe "event:goal filter validation" do
|
||||
test "valid filters", %{site: site} do
|
||||
insert(:goal, %{site: site, event_name: "Signup"})
|
||||
insert(:goal, %{site: site, page_path: "/thank-you"})
|
||||
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
"filters" => [
|
||||
["is", "event:goal", ["Signup", "Visit /thank-you"]]
|
||||
]
|
||||
}
|
||||
|> check_success(site, %{
|
||||
metrics: [:visitors],
|
||||
date_range: @date_range,
|
||||
filters: [
|
||||
[:is, "event:goal", [{:event, "Signup"}, {:page, "/thank-you"}]]
|
||||
],
|
||||
dimensions: [],
|
||||
order_by: nil,
|
||||
timezone: site.timezone,
|
||||
imported_data_requested: false
|
||||
})
|
||||
end
|
||||
|
||||
test "invalid event filter", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
"filters" => [
|
||||
["is", "event:goal", ["Signup"]]
|
||||
]
|
||||
}
|
||||
|> check_error(site, ~r/The goal `Signup` is not configured for this site/)
|
||||
end
|
||||
|
||||
test "invalid page filter", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
"filters" => [
|
||||
["is", "event:goal", ["Visit /thank-you"]]
|
||||
]
|
||||
}
|
||||
|> check_error(site, ~r/The goal `Visit \/thank-you` is not configured for this site/)
|
||||
end
|
||||
end
|
||||
|
||||
@ -235,139 +307,288 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
|
||||
|
||||
describe "dimensions validation" do
|
||||
for dimension <- Filters.event_props() do
|
||||
test "event:#{dimension} dimension" do
|
||||
test "event:#{dimension} dimension", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
"dimensions" => ["event:#{unquote(dimension)}"]
|
||||
}
|
||||
|> check_success(%{
|
||||
|> check_success(site, %{
|
||||
metrics: [:visitors],
|
||||
date_range: "all",
|
||||
date_range: @date_range,
|
||||
filters: [],
|
||||
dimensions: ["event:#{unquote(dimension)}"],
|
||||
order_by: nil
|
||||
order_by: nil,
|
||||
timezone: site.timezone,
|
||||
imported_data_requested: false
|
||||
})
|
||||
end
|
||||
end
|
||||
|
||||
for dimension <- Filters.visit_props() do
|
||||
test "visit:#{dimension} dimension" do
|
||||
test "visit:#{dimension} dimension", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
"dimensions" => ["visit:#{unquote(dimension)}"]
|
||||
}
|
||||
|> check_success(%{
|
||||
|> check_success(site, %{
|
||||
metrics: [:visitors],
|
||||
date_range: "all",
|
||||
date_range: @date_range,
|
||||
filters: [],
|
||||
dimensions: ["visit:#{unquote(dimension)}"],
|
||||
order_by: nil
|
||||
order_by: nil,
|
||||
timezone: site.timezone,
|
||||
imported_data_requested: false
|
||||
})
|
||||
end
|
||||
end
|
||||
|
||||
test "custom properties dimension" do
|
||||
test "custom properties dimension", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
"dimensions" => ["event:props:foobar"]
|
||||
}
|
||||
|> check_success(%{
|
||||
|> check_success(site, %{
|
||||
metrics: [:visitors],
|
||||
date_range: "all",
|
||||
date_range: @date_range,
|
||||
filters: [],
|
||||
dimensions: ["event:props:foobar"],
|
||||
order_by: nil
|
||||
order_by: nil,
|
||||
timezone: site.timezone,
|
||||
imported_data_requested: false
|
||||
})
|
||||
end
|
||||
|
||||
test "invalid dimension name passed" do
|
||||
test "invalid custom property dimension", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
"dimensions" => ["event:props:"]
|
||||
}
|
||||
|> check_error(site, ~r/Invalid dimensions/)
|
||||
end
|
||||
|
||||
test "invalid dimension name passed", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
"dimensions" => ["visitors"]
|
||||
}
|
||||
|> check_error(~r/Invalid dimensions/)
|
||||
|> check_error(site, ~r/Invalid dimensions/)
|
||||
end
|
||||
|
||||
test "invalid dimension" do
|
||||
test "invalid dimension", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
"dimensions" => "foobar"
|
||||
}
|
||||
|> check_error(~r/Invalid dimensions/)
|
||||
|> check_error(site, ~r/Invalid dimensions/)
|
||||
end
|
||||
|
||||
test "dimensions are not unique" do
|
||||
test "dimensions are not unique", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
"dimensions" => ["event:name", "event:name"]
|
||||
}
|
||||
|> check_error(~r/Some dimensions are listed multiple times/)
|
||||
|> check_error(site, ~r/Some dimensions are listed multiple times/)
|
||||
end
|
||||
end
|
||||
|
||||
describe "order_by validation" do
|
||||
test "ordering by metric" do
|
||||
test "ordering by metric", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["visitors", "events"],
|
||||
"date_range" => "all",
|
||||
"order_by" => [["events", "desc"], ["visitors", "asc"]]
|
||||
}
|
||||
|> check_success(%{
|
||||
|> check_success(site, %{
|
||||
metrics: [:visitors, :events],
|
||||
date_range: "all",
|
||||
date_range: @date_range,
|
||||
filters: [],
|
||||
dimensions: [],
|
||||
order_by: [{:events, :desc}, {:visitors, :asc}]
|
||||
order_by: [{:events, :desc}, {:visitors, :asc}],
|
||||
timezone: site.timezone,
|
||||
imported_data_requested: false
|
||||
})
|
||||
end
|
||||
|
||||
test "ordering by dimension" do
|
||||
test "ordering by dimension", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
"dimensions" => ["event:name"],
|
||||
"order_by" => [["event:name", "desc"]]
|
||||
}
|
||||
|> check_success(%{
|
||||
|> check_success(site, %{
|
||||
metrics: [:visitors],
|
||||
date_range: "all",
|
||||
date_range: @date_range,
|
||||
filters: [],
|
||||
dimensions: ["event:name"],
|
||||
order_by: [{"event:name", :desc}]
|
||||
order_by: [{"event:name", :desc}],
|
||||
timezone: site.timezone,
|
||||
imported_data_requested: false
|
||||
})
|
||||
end
|
||||
|
||||
test "ordering by invalid value" do
|
||||
test "ordering by invalid value", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
"order_by" => [["visssss", "desc"]]
|
||||
}
|
||||
|> check_error(~r/Invalid order_by entry/)
|
||||
|> check_error(site, ~r/Invalid order_by entry/)
|
||||
end
|
||||
|
||||
test "ordering by not queried metric" do
|
||||
test "ordering by not queried metric", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
"order_by" => [["events", "desc"]]
|
||||
}
|
||||
|> check_error(~r/Entry is not a queried metric or dimension/)
|
||||
|> check_error(site, ~r/Entry is not a queried metric or dimension/)
|
||||
end
|
||||
|
||||
test "ordering by not queried dimension" do
|
||||
test "ordering by not queried dimension", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
"order_by" => [["event:name", "desc"]]
|
||||
}
|
||||
|> check_error(~r/Entry is not a queried metric or dimension/)
|
||||
|> check_error(site, ~r/Entry is not a queried metric or dimension/)
|
||||
end
|
||||
end
|
||||
|
||||
describe "custom props access" do
|
||||
test "error if invalid filter", %{site: site, user: user} do
|
||||
ep =
|
||||
insert(:enterprise_plan, features: [Plausible.Billing.Feature.StatsAPI], user_id: user.id)
|
||||
|
||||
insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)
|
||||
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
"filters" => [["is", "event:props:foobar", ["foo"]]]
|
||||
}
|
||||
|> check_error(
|
||||
site,
|
||||
~r/The owner of this site does not have access to the custom properties feature/
|
||||
)
|
||||
end
|
||||
|
||||
test "error if invalid dimension", %{site: site, user: user} do
|
||||
ep =
|
||||
insert(:enterprise_plan, features: [Plausible.Billing.Feature.StatsAPI], user_id: user.id)
|
||||
|
||||
insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)
|
||||
|
||||
%{
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
"dimensions" => ["event:props:foobar"]
|
||||
}
|
||||
|> check_error(
|
||||
site,
|
||||
~r/The owner of this site does not have access to the custom properties feature/
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
describe "conversion_rate metric" do
|
||||
test "fails validation on its own", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["conversion_rate"],
|
||||
"date_range" => "all"
|
||||
}
|
||||
|> check_error(
|
||||
site,
|
||||
~r/Metric `conversion_rate` can only be queried with event:goal filters or dimensions/
|
||||
)
|
||||
end
|
||||
|
||||
test "succeeds with event:goal filter", %{site: site} do
|
||||
insert(:goal, %{site: site, event_name: "Signup"})
|
||||
|
||||
%{
|
||||
"metrics" => ["conversion_rate"],
|
||||
"date_range" => "all",
|
||||
"filters" => [["is", "event:goal", ["Signup"]]]
|
||||
}
|
||||
|> check_success(site, %{
|
||||
metrics: [:conversion_rate],
|
||||
date_range: @date_range,
|
||||
filters: [[:is, "event:goal", [event: "Signup"]]],
|
||||
dimensions: [],
|
||||
order_by: nil,
|
||||
timezone: site.timezone,
|
||||
imported_data_requested: false
|
||||
})
|
||||
end
|
||||
|
||||
test "succeeds with event:goal dimension", %{site: site} do
|
||||
insert(:goal, %{site: site, event_name: "Signup"})
|
||||
|
||||
%{
|
||||
"metrics" => ["conversion_rate"],
|
||||
"date_range" => "all",
|
||||
"dimensions" => ["event:goal"]
|
||||
}
|
||||
|> check_success(site, %{
|
||||
metrics: [:conversion_rate],
|
||||
date_range: @date_range,
|
||||
filters: [],
|
||||
dimensions: ["event:goal"],
|
||||
order_by: nil,
|
||||
timezone: site.timezone,
|
||||
imported_data_requested: false
|
||||
})
|
||||
end
|
||||
end
|
||||
|
||||
describe "session metrics" do
|
||||
test "single session metric succeeds", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["bounce_rate"],
|
||||
"date_range" => "all",
|
||||
"dimensions" => ["visit:device"]
|
||||
}
|
||||
|> check_success(site, %{
|
||||
metrics: [:bounce_rate],
|
||||
date_range: @date_range,
|
||||
filters: [],
|
||||
dimensions: ["visit:device"],
|
||||
order_by: nil,
|
||||
timezone: site.timezone,
|
||||
imported_data_requested: false
|
||||
})
|
||||
end
|
||||
|
||||
test "fails if using session metric with event dimension", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["bounce_rate"],
|
||||
"date_range" => "all",
|
||||
"dimensions" => ["event:props:foo"]
|
||||
}
|
||||
|> check_error(
|
||||
site,
|
||||
"Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions"
|
||||
)
|
||||
end
|
||||
|
||||
test "fails if using session metric with event filter", %{site: site} do
|
||||
%{
|
||||
"metrics" => ["bounce_rate"],
|
||||
"date_range" => "all",
|
||||
"filters" => [["is", "event:props:foo", ["(none)"]]]
|
||||
}
|
||||
|> check_error(
|
||||
site,
|
||||
"Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions"
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
@ -5,62 +5,62 @@ defmodule Plausible.Stats.TableDeciderTest do
|
||||
import Plausible.Stats.TableDecider
|
||||
|
||||
test "events_join_sessions? with experimental_reduced_joins disabled" do
|
||||
assert not events_join_sessions?(make_query(false, %{}))
|
||||
assert not events_join_sessions?(make_query(false, %{name: "pageview"}))
|
||||
assert events_join_sessions?(make_query(false, %{source: "Google"}))
|
||||
assert events_join_sessions?(make_query(false, %{entry_page: "/"}))
|
||||
assert events_join_sessions?(make_query(false, %{exit_page: "/"}))
|
||||
assert not events_join_sessions?(make_query(false, []))
|
||||
assert not events_join_sessions?(make_query(false, ["event:name"]))
|
||||
assert events_join_sessions?(make_query(false, ["visit:source"]))
|
||||
assert events_join_sessions?(make_query(false, ["visit:entry_page"]))
|
||||
assert events_join_sessions?(make_query(false, ["visit:exit_page"]))
|
||||
end
|
||||
|
||||
test "events_join_sessions? with experimental_reduced_joins enabled" do
|
||||
assert not events_join_sessions?(make_query(true, %{}))
|
||||
assert not events_join_sessions?(make_query(true, %{name: "pageview"}))
|
||||
assert not events_join_sessions?(make_query(true, %{source: "Google"}))
|
||||
assert events_join_sessions?(make_query(true, %{entry_page: "/"}))
|
||||
assert events_join_sessions?(make_query(true, %{exit_page: "/"}))
|
||||
assert not events_join_sessions?(make_query(true, []))
|
||||
assert not events_join_sessions?(make_query(true, ["event:name"]))
|
||||
assert not events_join_sessions?(make_query(true, ["visit:source"]))
|
||||
assert events_join_sessions?(make_query(true, ["visit:entry_page"]))
|
||||
assert events_join_sessions?(make_query(true, ["visit:exit_page"]))
|
||||
end
|
||||
|
||||
describe "partition_metrics" do
|
||||
test "with no metrics or filters" do
|
||||
query = make_query(false, %{})
|
||||
query = make_query(false, [])
|
||||
|
||||
assert partition_metrics([], query) == {[], [], []}
|
||||
end
|
||||
|
||||
test "session-only metrics accordingly" do
|
||||
query = make_query(false, %{})
|
||||
query = make_query(false, [])
|
||||
|
||||
assert partition_metrics([:bounce_rate, :views_per_visit], query) ==
|
||||
{[], [:bounce_rate, :views_per_visit], []}
|
||||
end
|
||||
|
||||
test "event-only metrics accordingly" do
|
||||
query = make_query(false, %{})
|
||||
query = make_query(false, [])
|
||||
|
||||
assert partition_metrics([:total_revenue, :visitors], query) ==
|
||||
{[:total_revenue, :visitors], [], []}
|
||||
end
|
||||
|
||||
test "filters from both, event-only metrics" do
|
||||
query = make_query(false, %{name: "pageview", source: "Google"})
|
||||
query = make_query(false, ["event:name", "visit:source"])
|
||||
|
||||
assert partition_metrics([:total_revenue], query) == {[:total_revenue], [], []}
|
||||
end
|
||||
|
||||
test "filters from both, session-only metrics" do
|
||||
query = make_query(false, %{name: "pageview", source: "Google"})
|
||||
query = make_query(false, ["event:name", "visit:source"])
|
||||
|
||||
assert partition_metrics([:bounce_rate], query) == {[], [:bounce_rate], []}
|
||||
end
|
||||
|
||||
test "session filters but no session metrics" do
|
||||
query = make_query(false, %{source: "Google"})
|
||||
query = make_query(false, ["visit:source"])
|
||||
|
||||
assert partition_metrics([:total_revenue], query) == {[:total_revenue], [], []}
|
||||
end
|
||||
|
||||
test "sample_percent is added to both types of metrics" do
|
||||
query = make_query(false, %{})
|
||||
query = make_query(false, [])
|
||||
|
||||
assert partition_metrics([:total_revenue, :sample_percent], query) ==
|
||||
{[:total_revenue, :sample_percent], [], []}
|
||||
@ -73,14 +73,14 @@ defmodule Plausible.Stats.TableDeciderTest do
|
||||
end
|
||||
|
||||
test "other metrics put in its own result" do
|
||||
query = make_query(false, %{})
|
||||
query = make_query(false, [])
|
||||
|
||||
assert partition_metrics([:time_on_page, :percentage, :total_visitors], query) ==
|
||||
{[], [], [:time_on_page, :percentage, :total_visitors]}
|
||||
end
|
||||
|
||||
test "raises if unknown metric" do
|
||||
query = make_query(false, %{})
|
||||
query = make_query(false, [])
|
||||
|
||||
assert_raise ArgumentError, fn ->
|
||||
partition_metrics([:foobar], query)
|
||||
@ -90,7 +90,7 @@ defmodule Plausible.Stats.TableDeciderTest do
|
||||
|
||||
describe "partition_metrics with experimental_reduced_joins enabled" do
|
||||
test "metrics that can be calculated on either when event-only metrics" do
|
||||
query = make_query(true, %{})
|
||||
query = make_query(true, [])
|
||||
|
||||
assert partition_metrics([:total_revenue, :visitors], query) ==
|
||||
{[:total_revenue, :visitors], [], []}
|
||||
@ -99,7 +99,7 @@ defmodule Plausible.Stats.TableDeciderTest do
|
||||
end
|
||||
|
||||
test "metrics that can be calculated on either when session-only metrics" do
|
||||
query = make_query(true, %{})
|
||||
query = make_query(true, [])
|
||||
|
||||
assert partition_metrics([:bounce_rate, :visitors], query) ==
|
||||
{[], [:bounce_rate, :visitors], []}
|
||||
@ -109,56 +109,60 @@ defmodule Plausible.Stats.TableDeciderTest do
|
||||
end
|
||||
|
||||
test "metrics that can be calculated on either are biased to sessions" do
|
||||
query = make_query(true, %{})
|
||||
query = make_query(true, [])
|
||||
|
||||
assert partition_metrics([:bounce_rate, :total_revenue, :visitors], query) ==
|
||||
{[:total_revenue], [:bounce_rate, :visitors], []}
|
||||
end
|
||||
|
||||
test "sample_percent is handled with either metrics" do
|
||||
query = make_query(true, %{})
|
||||
query = make_query(true, [])
|
||||
|
||||
assert partition_metrics([:visitors, :sample_percent], query) ==
|
||||
{[], [:visitors, :sample_percent], []}
|
||||
end
|
||||
|
||||
test "metric can be calculated on either, but filtering on events" do
|
||||
query = make_query(true, %{name: "pageview"})
|
||||
query = make_query(true, ["event:name"])
|
||||
|
||||
assert partition_metrics([:visitors], query) == {[:visitors], [], []}
|
||||
end
|
||||
|
||||
test "metric can be calculated on either, but filtering on events and sessions" do
|
||||
query = make_query(true, %{name: "pageview", exit_page: "/"})
|
||||
query = make_query(true, ["event:name", "visit:exit_page"])
|
||||
|
||||
assert partition_metrics([:visitors], query) == {[], [:visitors], []}
|
||||
end
|
||||
|
||||
test "metric can be calculated on either, filtering on either" do
|
||||
query = make_query(true, %{source: "Google"})
|
||||
query = make_query(true, ["visit:source"])
|
||||
|
||||
assert partition_metrics([:visitors], query) == {[], [:visitors], []}
|
||||
end
|
||||
|
||||
test "metric can be calculated on either, filtering on sessions" do
|
||||
query = make_query(true, %{exit_page: "/"})
|
||||
query = make_query(true, ["visit:exit_page"])
|
||||
|
||||
assert partition_metrics([:visitors], query) == {[], [:visitors], []}
|
||||
end
|
||||
|
||||
test "breakdown value leans metric" do
|
||||
query = make_query(true, %{})
|
||||
test "query dimensions lean metric" do
|
||||
assert partition_metrics([:visitors], make_query(true, [], ["event:name"])) ==
|
||||
{[:visitors], [], []}
|
||||
|
||||
assert partition_metrics([:visitors], query, "event:name") == {[:visitors], [], []}
|
||||
assert partition_metrics([:visitors], query, "visit:source") == {[], [:visitors], []}
|
||||
assert partition_metrics([:visitors], query, "visit:exit_page") == {[], [:visitors], []}
|
||||
assert partition_metrics([:visitors], make_query(true, [], ["visit:source"])) ==
|
||||
{[], [:visitors], []}
|
||||
|
||||
assert partition_metrics([:visitors], make_query(true, [], ["visit:exit_page"])) ==
|
||||
{[], [:visitors], []}
|
||||
end
|
||||
end
|
||||
|
||||
defp make_query(experimental_reduced_joins?, filters) do
|
||||
defp make_query(experimental_reduced_joins?, filter_keys, dimensions \\ []) do
|
||||
Query.from(build(:site), %{
|
||||
"experimental_reduced_joins" => to_string(experimental_reduced_joins?),
|
||||
"filters" => Jason.encode!(filters)
|
||||
"filters" => Enum.map(filter_keys, fn filter_key -> ["is", filter_key, []] end),
|
||||
"dimensions" => dimensions
|
||||
})
|
||||
end
|
||||
end
|
||||
|
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user