plausible/analytics
1
1
Fork 0
mirror of https://github.com/plausible/analytics.git synced 2024-08-17 05:40:24 +03:00
Code Issues Packages Projects Releases Wiki Activity

APIv2: Replace breakdown module with QueryBuilder (#4283)

Browse Source 
* WIP: Breakdown using QueryBuilder

* Revert "Remove problematic test"

This reverts commit b442bb5d1f.

* Get more breakdown tests passing

* Preload goals, sort when dealing with time_on_page

* Handle conversion_rate in breakdowns

* Simplify ordering by using selected_as consistently for dimensions

* Get breakdown tests passing

* Strings to atoms in keys for StatsController.transform_keys calls to work

* Handle revenue metrics removal

* Add test for nil-removal case

* Include percentage metric

* Fix and test with imported locations

* Fixup time-on-page

* Fix country/region automatic filters

* Handle multiple imports (os/browser version) in importsv2

* Filter goals

* Default to ordering by page as well

* Calculate conversion rate on sessions if needed

* Order by event dimensions - handles event:page special case

* Update tests

* Update more tests, handle goal=0 case in imports

* Handle event:goal breakdowns correctly with filters

* Revenue to money

* Improved table deciding

* Also update event:page filters on event:page breakdown

* bounce_rate to 0

Previous behavior relied on two queries being made - new query leads to 0 naturally

* Update pagination test

* dont count non-pageviews as path goal completions

* Make revenue logic breakdown-specific

Its hard to fit into the new schema and likely needs a rethink for apiv2

* Retain previous behavior for TimeSeries module

* Get GA4 test passing

Most failures are related to ordering, pageviews shouldnt be read off of sessions

* Clean up old methods

* Simplify imported.ex

* Dont crash on garbage filters

* Reflect ordering-related change in test

* Fix test data

* Update table_decider

* Re-simplify get_revenue_tracking_currency

* Revert revenue changes

* Use Query.set

* Remove a TODO

* csv importer: no pageviews

Pageviews were incorrectly fetched from sessions table before, causing issues

* csv importer tweaking

* Remove use Plausible

* to_existing_atom
This commit is contained in:
Karl-Aksel Puulmann 2024-07-01 09:03:33 +03:00 committed by GitHub
parent f4e091452c
commit 7dd12d1dd6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
25 changed files with 660 additions and 1123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -16,6 +16,7 @@ All notable changes to this project will be documented in this file.
- Increase hourly request limit for API keys in CE from 600 to 1000000 (practically removing the limit) plausible/analytics#4200
- Make TCP connections try IPv6 first with IPv4 fallback in CE plausible/analytics#4245
- `is` and `is not` filters in dashboard no longer support wildcards. Use contains/does not contain filter instead.
- `bounce_rate` metric now returns 0 instead of null for event:page breakdown when page has never been entry page.
### Fixed

2
lib/plausible/stats/base.ex
View File

@ -250,6 +250,8 @@ defmodule Plausible.Stats.Base do
end
defp select_session_metric(:percentage, _query), do: %{}
defp select_session_metric(:conversion_rate, _query), do: %{}
defp select_session_metric(:group_conversion_rate, _query), do: %{}
def filter_converted_sessions(db_query, site, query) do
if Query.has_event_filters?(query) do

784
lib/plausible/stats/breakdown.ex
View File

@ -3,344 +3,61 @@ defmodule Plausible.Stats.Breakdown do
use Plausible
use Plausible.Stats.Fragments
import Plausible.Stats.{Base, Imported}
import Plausible.Stats.Base
import Ecto.Query
alias Plausible.Stats.{Query, Util, TableDecider}
alias Plausible.Stats.{Query, QueryOptimizer, QueryResult, SQL}
alias Plausible.Stats.Filters.QueryParser
@no_ref "Direct / None"
@not_set "(not set)"
def breakdown(site, %Query{dimensions: [dimension]} = query, metrics, pagination, _opts \\ []) do
transformed_metrics = transform_metrics(metrics, dimension)
@session_metrics [:bounce_rate, :visit_duration]
@revenue_metrics on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: [])
@event_metrics [:visits, :visitors, :pageviews, :events, :percentage] ++ @revenue_metrics
# These metrics can be asked from the `breakdown/5` function,
# but they are different from regular metrics such as `visitors`,
# or `bounce_rate` - we cannot currently "select them" directly in
# the db queries. Instead, we need to artificially append them to
# the breakdown results later on.
@computed_metrics [:conversion_rate, :total_visitors]
def breakdown(site, query, metrics, pagination, opts \\ [])
def breakdown(
site,
%Query{dimensions: ["event:goal"]} = query,
metrics,
pagination,
opts
) do
site = Plausible.Repo.preload(site, :goals)
{event_goals, pageview_goals} = Enum.split_with(site.goals, & &1.event_name)
events = Enum.map(event_goals, & &1.event_name)
event_query =
query
|> Query.put_filter([:is, "event:name", events])
|> Query.set_dimensions(["event:name"])
if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)
no_revenue = {nil, metrics -- @revenue_metrics}
{revenue_goals, metrics} =
on_ee do
if Plausible.Billing.Feature.RevenueGoals.enabled?(site) do
revenue_goals = Enum.filter(event_goals, &Plausible.Goal.Revenue.revenue?/1)
metrics = if Enum.empty?(revenue_goals), do: metrics -- @revenue_metrics, else: metrics
{revenue_goals, metrics}
else
no_revenue
end
else
no_revenue
end
metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics
event_q =
if Enum.any?(event_goals) do
site
|> breakdown_events(event_query, metrics_to_select)
|> apply_pagination(pagination)
else
nil
end
page_q =
if Enum.any?(pageview_goals) do
page_query = Query.set_dimensions(query, ["event:page"])
page_exprs = Enum.map(pageview_goals, & &1.page_path)
page_regexes = Enum.map(page_exprs, &page_regex/1)
select_columns = metrics_to_select |> select_event_metrics |> mark_revenue_as_nil
from(e in base_event_query(site, page_query),
order_by: [desc: fragment("uniq(?)", e.user_id)],
where:
fragment(
"notEmpty(multiMatchAllIndices(?, ?) as indices)",
e.pathname,
^page_regexes
) and e.name == "pageview",
array_join: index in fragment("indices"),
group_by: index,
select: %{
name: fragment("concat('Visit ', ?[?])", ^page_exprs, index)
}
)
|> select_merge(^select_columns)
|> merge_imported_pageview_goals(site, page_query, page_exprs, metrics_to_select)
|> apply_pagination(pagination)
else
nil
end
full_q =
case {event_q, page_q} do
{nil, nil} ->
nil
{event_q, nil} ->
event_q
{nil, page_q} ->
page_q
{event_q, page_q} ->
from(
e in subquery(union_all(event_q, ^page_q)),
order_by: [desc: e.visitors]
)
|> apply_pagination(pagination)
end
if full_q do
full_q
|> maybe_add_conversion_rate(site, query, metrics)
|> ClickhouseRepo.all()
|> transform_keys(%{name: :goal})
|> cast_revenue_metrics_to_money(revenue_goals)
|> Util.keep_requested_metrics(metrics)
else
[]
end
end
def breakdown(
site,
%Query{dimensions: ["event:props:" <> custom_prop]} = query,
metrics,
pagination,
opts
) do
{currency, metrics} =
on_ee do
Plausible.Stats.Goal.Revenue.get_revenue_tracking_currency(site, query, metrics)
else
{nil, metrics}
end
metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics
if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)
breakdown_events(site, query, metrics_to_select)
|> maybe_add_conversion_rate(site, query, metrics)
|> paginate_and_execute(metrics, pagination)
|> transform_keys(%{breakdown_prop_value: custom_prop})
|> Enum.map(&cast_revenue_metrics_to_money(&1, currency))
end
def breakdown(site, %Query{dimensions: ["event:page"]} = query, metrics, pagination, opts) do
event_metrics =
metrics
|> Util.maybe_add_visitors_metric()
|> Enum.filter(&(&1 in @event_metrics))
if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)
event_result =
site
|> breakdown_events(query, event_metrics)
|> maybe_add_group_conversion_rate(&breakdown_events/3, site, query, metrics)
|> paginate_and_execute(metrics, pagination)
|> maybe_add_time_on_page(site, query, metrics)
session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))
entry_page_query =
case event_result do
[] ->
query
pages ->
query
|> Query.remove_filters(["event:page"])
|> Query.put_filter([:is, "visit:entry_page", Enum.map(pages, & &1[:page])])
|> Query.set_dimensions(["visit:entry_page"])
end
if Enum.any?(event_metrics) && Enum.empty?(event_result) do
[]
else
{limit, _page} = pagination
session_result =
breakdown_sessions(site, entry_page_query, session_metrics)
|> paginate_and_execute(session_metrics, {limit, 1})
|> transform_keys(%{entry_page: :page})
metrics = metrics ++ [:page]
zip_results(
event_result,
session_result,
:page,
metrics
query_with_metrics =
Query.set(
query,
metrics: transformed_metrics,
order_by: infer_order_by(transformed_metrics, dimension),
dimensions: transform_dimensions(dimension),
filters: query.filters ++ dimension_filters(dimension),
preloaded_goals: QueryParser.preload_goals_if_needed(site, query.filters, [dimension]),
v2: true
)
|> Enum.map(&Map.take(&1, metrics))
end
end
|> QueryOptimizer.optimize()
def breakdown(site, %Query{dimensions: ["event:name"]} = query, metrics, pagination, opts) do
if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)
q = SQL.QueryBuilder.build(query_with_metrics, site)
breakdown_events(site, query, metrics)
|> paginate_and_execute(metrics, pagination)
end
def breakdown(site, query, metrics, pagination, opts) do
query = maybe_update_breakdown_filters(query)
if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)
metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics
case breakdown_table(query, metrics) do
:session ->
breakdown_sessions(site, query, metrics_to_select)
|> maybe_add_group_conversion_rate(&breakdown_sessions/3, site, query, metrics)
|> paginate_and_execute(metrics, pagination)
:event ->
breakdown_events(site, query, metrics_to_select)
|> maybe_add_group_conversion_rate(&breakdown_events/3, site, query, metrics)
|> paginate_and_execute(metrics, pagination)
end
end
defp maybe_update_breakdown_filters(%Query{dimensions: [visit_entry_prop]} = query)
when visit_entry_prop in [
"visit:source",
"visit:entry_page",
"visit:utm_medium",
"visit:utm_source",
"visit:utm_campaign",
"visit:utm_content",
"visit:utm_term",
"visit:entry_page",
"visit:referrer"
] do
update_hostname_filter_prop(query, "visit:entry_page_hostname")
end
defp maybe_update_breakdown_filters(%Query{dimensions: ["visit:exit_page"]} = query) do
update_hostname_filter_prop(query, "visit:exit_page_hostname")
end
defp maybe_update_breakdown_filters(query) do
query
end
defp update_hostname_filter_prop(query, visit_prop) do
case Query.get_filter(query, "event:hostname") do
nil ->
query
[op, "event:hostname", value] ->
query
|> Query.put_filter([op, visit_prop, value])
end
end
# Backwards compatibility
defp breakdown_table(%Query{experimental_reduced_joins?: false}, _), do: :session
defp breakdown_table(%Query{dimensions: ["visit:entry_page"]}, _metrics), do: :session
defp breakdown_table(%Query{dimensions: ["visit:entry_page_hostname"]}, _metrics), do: :session
defp breakdown_table(%Query{dimensions: ["visit:exit_page"]}, _metrics), do: :session
defp breakdown_table(%Query{dimensions: ["visit:exit_page_hostname"]}, _metrics), do: :session
defp breakdown_table(%Query{dimensions: [_dimension]} = query, metrics) do
{_, session_metrics, _} = TableDecider.partition_metrics(metrics, query)
if not Enum.empty?(session_metrics) do
:session
else
:event
end
end
defp zip_results(event_result, session_result, property, metrics) do
null_row = Enum.map(metrics, fn metric -> {metric, nil} end) |> Enum.into(%{})
prop_values =
Enum.map(event_result ++ session_result, fn row -> row[property] end)
|> Enum.uniq()
Enum.map(prop_values, fn value ->
event_row = Enum.find(event_result, fn row -> row[property] == value end) || %{}
session_row = Enum.find(session_result, fn row -> row[property] == value end) || %{}
null_row
|> Map.merge(event_row)
|> Map.merge(session_row)
end)
|> sort_results(metrics)
end
defp breakdown_sessions(site, %Query{dimensions: [dimension]} = query, metrics) do
from(s in query_sessions(site, query),
order_by: [desc: fragment("uniq(?)", s.user_id)],
select: ^select_session_metrics(metrics, query)
)
|> filter_converted_sessions(site, query)
|> do_group_by(dimension)
|> merge_imported(site, query, metrics)
|> add_percentage_metric(site, query, metrics)
end
defp breakdown_events(site, %Query{dimensions: [dimension]} = query, metrics) do
from(e in base_event_query(site, query),
order_by: [desc: fragment("uniq(?)", e.user_id)],
select: %{}
)
|> do_group_by(dimension)
|> select_merge(^select_event_metrics(metrics))
|> merge_imported(site, query, metrics)
|> add_percentage_metric(site, query, metrics)
end
defp paginate_and_execute(_, [], _), do: []
defp paginate_and_execute(q, metrics, pagination) do
q
|> apply_pagination(pagination)
|> ClickhouseRepo.all()
|> Util.keep_requested_metrics(metrics)
|> QueryResult.from(query_with_metrics)
|> build_breakdown_result(query_with_metrics, metrics)
|> maybe_add_time_on_page(site, query_with_metrics, metrics)
|> update_currency_metrics(site, query_with_metrics)
end
defp build_breakdown_result(query_result, query, metrics) do
query_result.results
|> Enum.map(fn %{dimensions: dimensions, metrics: entry_metrics} ->
dimension_map =
query.dimensions |> Enum.map(&result_key/1) |> Enum.zip(dimensions) |> Enum.into(%{})
metrics_map = Enum.zip(metrics, entry_metrics) |> Enum.into(%{})
Map.merge(dimension_map, metrics_map)
end)
end
defp result_key("event:props:" <> custom_property), do: custom_property
defp result_key("event:" <> key), do: key |> String.to_existing_atom()
defp result_key("visit:" <> key), do: key |> String.to_existing_atom()
defp result_key(dimension), do: dimension
defp maybe_add_time_on_page(event_results, site, query, metrics) do
if :time_on_page in metrics do
if query.dimensions == ["event:page"] and :time_on_page in metrics do
pages = Enum.map(event_results, & &1[:page])
time_on_page_result = breakdown_time_on_page(site, query, pages)
Enum.map(event_results, fn row ->
event_results
|> Enum.map(fn row ->
Map.put(row, :time_on_page, time_on_page_result[row[:page]])
end)
else
@ -431,346 +148,60 @@ defmodule Plausible.Stats.Breakdown do
|> Map.new()
end
defp do_group_by(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,
"event:props:" <> prop
) do
from(
e in q,
select_merge: %{
breakdown_prop_value:
selected_as(
fragment(
"if(not empty(?), ?, '(none)')",
get_by_key(e, :meta, ^prop),
get_by_key(e, :meta, ^prop)
),
:breakdown_prop_value
)
},
group_by: selected_as(:breakdown_prop_value),
order_by: {:asc, selected_as(:breakdown_prop_value)}
)
end
defp transform_metrics(metrics, dimension) do
metrics =
if is_nil(metric_to_order_by(metrics)) do
metrics ++ [:visitors]
else
metrics
end
defp do_group_by(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,
"event:name"
) do
from(
e in q,
group_by: e.name,
select_merge: %{name: e.name},
order_by: {:asc, e.name}
)
end
defp do_group_by(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,
"event:page"
) do
from(
e in q,
group_by: e.pathname,
select_merge: %{page: e.pathname},
order_by: {:asc, e.pathname}
)
end
defp do_group_by(q, "visit:source") do
from(
s in q,
group_by: s.source,
select_merge: %{
source: fragment("if(empty(?), ?, ?)", s.source, @no_ref, s.source)
},
order_by: {:asc, s.source}
)
end
defp do_group_by(q, "visit:country") do
from(
s in q,
where: s.country != "\0\0" and s.country != "ZZ",
group_by: s.country,
select_merge: %{country: s.country},
order_by: {:asc, s.country}
)
end
defp do_group_by(q, "visit:region") do
from(
s in q,
where: s.region != "",
group_by: s.region,
select_merge: %{region: s.region},
order_by: {:asc, s.region}
)
end
defp do_group_by(q, "visit:city") do
from(
s in q,
where: s.city != 0,
group_by: s.city,
select_merge: %{city: s.city},
order_by: {:asc, s.city}
)
end
defp do_group_by(q, "visit:entry_page") do
from(
s in q,
# Sessions without pageviews don't get entry_page assigned, hence they should get ignored
where: s.entry_page != "",
group_by: s.entry_page,
select_merge: %{entry_page: s.entry_page},
order_by: {:asc, s.entry_page}
)
end
defp do_group_by(q, "visit:exit_page") do
from(
s in q,
# Sessions without pageviews don't get entry_page assigned, hence they should get ignored
where: s.entry_page != "",
group_by: s.exit_page,
select_merge: %{exit_page: s.exit_page},
order_by: {:asc, s.exit_page}
)
end
defp do_group_by(q, "visit:referrer") do
from(
s in q,
group_by: s.referrer,
select_merge: %{
referrer: fragment("if(empty(?), ?, ?)", s.referrer, @no_ref, s.referrer)
},
order_by: {:asc, s.referrer}
)
end
defp do_group_by(q, "visit:utm_medium") do
from(
s in q,
where: fragment("not empty(?)", s.utm_medium),
group_by: s.utm_medium,
select_merge: %{
utm_medium: s.utm_medium
}
)
end
defp do_group_by(q, "visit:utm_source") do
from(
s in q,
where: fragment("not empty(?)", s.utm_source),
group_by: s.utm_source,
select_merge: %{
utm_source: s.utm_source
}
)
end
defp do_group_by(q, "visit:utm_campaign") do
from(
s in q,
where: fragment("not empty(?)", s.utm_campaign),
group_by: s.utm_campaign,
select_merge: %{
utm_campaign: s.utm_campaign
}
)
end
defp do_group_by(q, "visit:utm_content") do
from(
s in q,
where: fragment("not empty(?)", s.utm_content),
group_by: s.utm_content,
select_merge: %{
utm_content: s.utm_content
}
)
end
defp do_group_by(q, "visit:utm_term") do
from(
s in q,
where: fragment("not empty(?)", s.utm_term),
group_by: s.utm_term,
select_merge: %{
utm_term: s.utm_term
}
)
end
defp do_group_by(q, "visit:device") do
from(
s in q,
group_by: s.device,
select_merge: %{
device: fragment("if(empty(?), ?, ?)", s.device, @not_set, s.device)
},
order_by: {:asc, s.device}
)
end
defp do_group_by(q, "visit:os") do
from(
s in q,
group_by: s.os,
select_merge: %{
os: fragment("if(empty(?), ?, ?)", s.os, @not_set, s.os)
},
order_by: {:asc, s.os}
)
end
defp do_group_by(q, "visit:os_version") do
from(
s in q,
group_by: [s.os, s.os_version],
select_merge: %{
os: fragment("if(empty(?), ?, ?)", s.os, @not_set, s.os),
os_version:
fragment(
"if(empty(?), ?, ?)",
s.os_version,
@not_set,
s.os_version
)
},
order_by: {:asc, s.os_version}
)
end
defp do_group_by(q, "visit:browser") do
from(
s in q,
group_by: s.browser,
select_merge: %{
browser: fragment("if(empty(?), ?, ?)", s.browser, @not_set, s.browser)
},
order_by: {:asc, s.browser}
)
end
defp do_group_by(q, "visit:browser_version") do
from(
s in q,
group_by: [s.browser, s.browser_version],
select_merge: %{
browser: fragment("if(empty(?), ?, ?)", s.browser, @not_set, s.browser),
browser_version:
fragment("if(empty(?), ?, ?)", s.browser_version, @not_set, s.browser_version)
},
order_by: {:asc, s.browser_version}
)
end
defp group_by_field_names("event:props:" <> _prop), do: [:name]
defp group_by_field_names("visit:os_version"), do: [:os, :os_version]
defp group_by_field_names("visit:browser_version"), do: [:browser, :browser_version]
defp group_by_field_names(property), do: [Plausible.Stats.Filters.without_prefix(property)]
defp on_matches_group_by(fields) do
Enum.reduce(fields, nil, &fields_equal/2)
end
defp outer_order_by(fields) do
Enum.map(fields, fn field_name -> {:asc, dynamic([q], field(q, ^field_name))} end)
end
defp fields_equal(field_name, nil),
do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name))
defp fields_equal(field_name, condition),
do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name) and ^condition)
defp sort_results(results, metrics) do
Enum.sort_by(
results,
fn entry ->
case entry[sorting_key(metrics)] do
nil -> 0
n -> n
end
end,
:desc
)
end
# This function injects a conversion_rate metric into
# a breakdown query. It is calculated as X / Y, where:
#
# * X is the number of conversions for a breakdown
# result (conversion = number of visitors who
# completed the filtered goal with the filtered
# custom properties).
#
# * Y is the number of all visitors for this breakdown
# result without the `event:goal` and `event:props:*`
# filters.
defp maybe_add_group_conversion_rate(
q,
breakdown_fn,
site,
%Query{dimensions: [dimension]} = query,
metrics
) do
if :conversion_rate in metrics do
breakdown_total_visitors_query =
query |> Query.remove_filters(["event:goal", "event:props"])
breakdown_total_visitors_q =
breakdown_fn.(site, breakdown_total_visitors_query, [:visitors])
from(e in subquery(q),
left_join: c in subquery(breakdown_total_visitors_q),
on: ^on_matches_group_by(group_by_field_names(dimension)),
select_merge: %{
total_visitors: c.visitors,
conversion_rate:
fragment(
"if(? > 0, round(? / ? * 100, 1), 0)",
c.visitors,
e.visitors,
c.visitors
)
},
order_by: [desc: e.visitors],
order_by: ^outer_order_by(group_by_field_names(dimension))
)
else
q
end
end
# When querying custom event goals and pageviewgoals together, UNION ALL is used
# so the same fields must be present on both sides of the union. This change to the
# query will ensure that we don't unnecessarily read revenue column for pageview goals
defp mark_revenue_as_nil(select_columns) do
select_columns
|> Map.replace(:total_revenue, nil)
|> Map.replace(:average_revenue, nil)
end
defp sorting_key(metrics) do
if Enum.member?(metrics, :visitors), do: :visitors, else: List.first(metrics)
end
defp transform_keys(results, keys_to_replace) do
Enum.map(results, fn map ->
Enum.map(map, fn {key, val} ->
{Map.get(keys_to_replace, key, key), val}
end)
|> Enum.into(%{})
Enum.map(metrics, fn metric ->
case {metric, dimension} do
{:conversion_rate, "event:props:" <> _} -> :conversion_rate
{:conversion_rate, "event:goal"} -> :conversion_rate
{:conversion_rate, _} -> :group_conversion_rate
_ -> metric
end
end)
end
defp infer_order_by(metrics, "event:goal"), do: [{metric_to_order_by(metrics), :desc}]
defp infer_order_by(metrics, dimension),
do: [{metric_to_order_by(metrics), :desc}, {dimension, :asc}]
defp metric_to_order_by(metrics) do
Enum.find(metrics, &(&1 != :time_on_page))
end
def transform_dimensions("visit:browser_version"),
do: ["visit:browser", "visit:browser_version"]
def transform_dimensions("visit:os_version"), do: ["visit:os", "visit:os_version"]
def transform_dimensions(dimension), do: [dimension]
@filter_dimensions_not %{
"visit:city" => [0],
"visit:country" => ["\0\0", "ZZ"],
"visit:region" => [""],
"visit:utm_medium" => [""],
"visit:utm_source" => [""],
"visit:utm_campaign" => [""],
"visit:utm_content" => [""],
"visit:utm_term" => [""],
"visit:entry_page" => [""],
"visit:exit_page" => [""]
}
@extra_filter_dimensions Map.keys(@filter_dimensions_not)
defp dimension_filters(dimension) when dimension in @extra_filter_dimensions do
[[:is_not, dimension, Map.get(@filter_dimensions_not, dimension)]]
end
defp dimension_filters(_), do: []
defp apply_pagination(q, {limit, page}) do
offset = (page - 1) * limit
@ -780,10 +211,39 @@ defmodule Plausible.Stats.Breakdown do
end
on_ee do
defp cast_revenue_metrics_to_money(results, revenue_goals) do
Plausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(results, revenue_goals)
defp update_currency_metrics(results, site, %Query{dimensions: ["event:goal"]}) do
site = Plausible.Repo.preload(site, :goals)
{event_goals, _pageview_goals} = Enum.split_with(site.goals, & &1.event_name)
revenue_goals = Enum.filter(event_goals, &Plausible.Goal.Revenue.revenue?/1)
if length(revenue_goals) > 0 and Plausible.Billing.Feature.RevenueGoals.enabled?(site) do
Plausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(results, revenue_goals)
else
remove_revenue_metrics(results)
end
end
defp update_currency_metrics(results, site, query) do
{currency, _metrics} =
Plausible.Stats.Goal.Revenue.get_revenue_tracking_currency(site, query, query.metrics)
if currency do
results
|> Enum.map(&Plausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(&1, currency))
else
remove_revenue_metrics(results)
end
end
else
defp cast_revenue_metrics_to_money(results, _revenue_goals), do: results
defp update_currency_metrics(results, _site, _query), do: remove_revenue_metrics(results)
end
defp remove_revenue_metrics(results) do
Enum.map(results, fn map ->
map
|> Map.delete(:total_revenue)
|> Map.delete(:average_revenue)
end)
end
end

20
lib/plausible/stats/filters/query_parser.ex
View File

@ -4,6 +4,7 @@ defmodule Plausible.Stats.Filters.QueryParser do
alias Plausible.Stats.TableDecider
alias Plausible.Stats.Filters
alias Plausible.Stats.Query
alias Plausible.Stats.Metrics
def parse(site, params, now \\ nil) when is_map(params) do
with {:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])),
@ -44,17 +45,12 @@ defmodule Plausible.Stats.Filters.QueryParser do
defp parse_metrics(_invalid_metrics), do: {:error, "Invalid metrics passed"}
defp parse_metric("time_on_page"), do: {:ok, :time_on_page}
defp parse_metric("conversion_rate"), do: {:ok, :conversion_rate}
defp parse_metric("group_conversion_rate"), do: {:ok, :group_conversion_rate}
defp parse_metric("visitors"), do: {:ok, :visitors}
defp parse_metric("pageviews"), do: {:ok, :pageviews}
defp parse_metric("events"), do: {:ok, :events}
defp parse_metric("visits"), do: {:ok, :visits}
defp parse_metric("bounce_rate"), do: {:ok, :bounce_rate}
defp parse_metric("visit_duration"), do: {:ok, :visit_duration}
defp parse_metric("views_per_visit"), do: {:ok, :views_per_visit}
defp parse_metric(unknown_metric), do: {:error, "Unknown metric '#{inspect(unknown_metric)}'"}
defp parse_metric(metric_str) do
case Metrics.from_string(metric_str) do
{:ok, metric} -> {:ok, metric}
_ -> {:error, "Unknown metric '#{inspect(metric_str)}'"}
end
end
def parse_filters(filters) when is_list(filters) do
parse_list(filters, &parse_filter/1)
@ -284,7 +280,7 @@ defmodule Plausible.Stats.Filters.QueryParser do
end
end
defp preload_goals_if_needed(site, filters, dimensions) do
def preload_goals_if_needed(site, filters, dimensions) do
goal_filters? =
Enum.any?(filters, fn [_, filter_key | _rest] -> filter_key == "event:goal" end)

11
lib/plausible/stats/filters/where_builder.ex
View File

@ -10,6 +10,8 @@ defmodule Plausible.Stats.Filters.WhereBuilder do
use Plausible.Stats.Fragments
require Logger
@sessions_only_visit_fields [
:entry_page,
:exit_page,
@ -150,6 +152,15 @@ defmodule Plausible.Stats.Filters.WhereBuilder do
true
end
defp add_filter(table, _query, filter) do
Logger.info("Unable to process garbage filter. No results are returned",
table: table,
filter: filter
)
false
end
defp filter_custom_prop(prop_name, column_name, [:is, _, values]) do
none_value_included = Enum.member?(values, "(none)")

77
lib/plausible/stats/imported/base.ex
View File

@ -54,10 +54,12 @@ defmodule Plausible.Stats.Imported.Base do
def property_to_table_mappings(), do: @property_to_table_mappings
def query_imported(site, query) do
query
|> transform_filters()
|> decide_table()
|> query_imported(site, query)
[table] =
query
|> transform_filters()
|> decide_tables()
query_imported(table, site, query)
end
def query_imported(table, site, query) do
@ -75,13 +77,13 @@ defmodule Plausible.Stats.Imported.Base do
|> apply_filter(query)
end
def decide_table(query) do
def decide_tables(query) do
query = transform_filters(query)
if custom_prop_query?(query) do
do_decide_custom_prop_table(query)
else
do_decide_table(query)
do_decide_tables(query)
end
end
@ -92,19 +94,17 @@ defmodule Plausible.Stats.Imported.Base do
[:is, "event:name", ["pageview"]] -> true
_ -> false
end)
|> Enum.flat_map(fn filter ->
case filter do
[op, "event:goal", events] ->
events
|> Enum.group_by(&elem(&1, 0), &elem(&1, 1))
|> Enum.map(fn
{:event, names} -> [op, "event:name", names]
{:page, pages} -> [op, "event:page", pages]
end)
|> Enum.flat_map(fn
[op, "event:goal", clauses] ->
clauses
|> Enum.group_by(&elem(&1, 0), &elem(&1, 1))
|> Enum.map(fn
{:event, names} -> [op, "event:name", names]
{:page, pages} -> [op, "event:page", pages]
end)
filter ->
[filter]
end
filter ->
[filter]
end)
struct!(query, filters: new_filters)
@ -136,10 +136,10 @@ defmodule Plausible.Stats.Imported.Base do
do_decide_custom_prop_table(query, custom_prop_filter)
_ ->
nil
[]
end
else
nil
[]
end
end
@ -158,23 +158,27 @@ defmodule Plausible.Stats.Imported.Base do
end)
if has_required_name_filter? and not has_unsupported_filters? do
"imported_custom_events"
["imported_custom_events"]
else
nil
[]
end
end
defp do_decide_table(%Query{filters: [], dimensions: []}), do: "imported_visitors"
defp do_decide_tables(%Query{filters: [], dimensions: []}), do: ["imported_visitors"]
defp do_decide_table(%Query{filters: [], dimensions: ["event:goal"]}) do
"imported_custom_events"
defp do_decide_tables(%Query{filters: [], dimensions: ["event:goal"]}) do
["imported_pages", "imported_custom_events"]
end
defp do_decide_table(%Query{filters: [], dimensions: [dimension]}) do
@property_to_table_mappings[dimension]
defp do_decide_tables(%Query{filters: [], dimensions: [dimension]}) do
if Map.has_key?(@property_to_table_mappings, dimension) do
[@property_to_table_mappings[dimension]]
else
[]
end
end
defp do_decide_table(%Query{filters: filters, dimensions: ["event:goal"]}) do
defp do_decide_tables(%Query{filters: filters, dimensions: ["event:goal"]}) do
filter_props = Enum.map(filters, &Enum.at(&1, 1))
any_event_name_filters? = "event:name" in filter_props
@ -182,27 +186,28 @@ defmodule Plausible.Stats.Imported.Base do
any_other_filters? = Enum.any?(filter_props, &(&1 not in ["event:page", "event:name"]))
cond do
any_other_filters? -> nil
any_event_name_filters? and not any_page_filters? -> "imported_custom_events"
any_page_filters? and not any_event_name_filters? -> "imported_pages"
true -> nil
any_other_filters? -> []
any_event_name_filters? and not any_page_filters? -> ["imported_custom_events"]
any_page_filters? and not any_event_name_filters? -> ["imported_pages"]
true -> []
end
end
defp do_decide_table(%Query{filters: filters, dimensions: dimensions}) do
defp do_decide_tables(%Query{filters: filters, dimensions: dimensions}) do
table_candidates =
filters
|> Enum.map(fn [_, filter_key | _] -> filter_key end)
|> Enum.concat(dimensions)
|> Enum.map(fn
"visit:screen" -> "visit:device"
prop -> prop
dimension -> dimension
end)
|> Enum.map(&@property_to_table_mappings[&1])
case Enum.uniq(table_candidates) do
[candidate] -> candidate
_ -> nil
[nil] -> []
[candidate] -> [candidate]
_ -> []
end
end

366
lib/plausible/stats/imported/imported.ex
View File

@ -4,10 +4,11 @@ defmodule Plausible.Stats.Imported do
import Ecto.Query
import Plausible.Stats.Fragments
import Plausible.Stats.Util, only: [shortname: 2]
alias Plausible.Stats.Base
alias Plausible.Stats.Imported
alias Plausible.Stats.Query
alias Plausible.Stats.SQL.QueryBuilder
@no_ref "Direct / None"
@not_set "(not set)"
@ -15,7 +16,7 @@ defmodule Plausible.Stats.Imported do
@property_to_table_mappings Imported.Base.property_to_table_mappings()
@imported_properties Map.keys(@property_to_table_mappings) ++
@imported_dimensions Map.keys(@property_to_table_mappings) ++
Plausible.Imported.imported_custom_props()
@goals_with_url Plausible.Imported.goals_with_url()
@ -37,7 +38,7 @@ defmodule Plausible.Stats.Imported do
(see `@goals_with_url` and `@goals_with_path`).
"""
def schema_supports_query?(query) do
not is_nil(Imported.Base.decide_table(query))
length(Imported.Base.decide_tables(query)) > 0
end
def merge_imported_country_suggestions(native_q, _site, %Plausible.Stats.Query{
@ -267,78 +268,6 @@ defmodule Plausible.Stats.Imported do
def merge_imported(q, _, %Query{include_imported: false}, _), do: q
# Note: Only called for APIv2, old APIs use merge_imported_pageview_goals
def merge_imported(q, site, %Query{dimensions: ["event:goal"]} = query, metrics)
when query.v2 do
{events, page_regexes} = Filters.Utils.split_goals_query_expressions(query.preloaded_goals)
events_q =
"imported_custom_events"
|> Imported.Base.query_imported(site, query)
|> where([i], i.visitors > 0)
|> select_merge([i], %{
dim0: selected_as(fragment("-indexOf(?, ?)", ^events, i.name), :dim0)
})
|> select_imported_metrics(metrics)
|> group_by([], selected_as(:dim0))
|> where([], selected_as(:dim0) != 0)
pages_q =
"imported_pages"
|> Imported.Base.query_imported(site, query)
|> where([i], i.visitors > 0)
|> where(
[i],
fragment("notEmpty(multiMatchAllIndices(?, ?) as indices)", i.page, ^page_regexes)
)
|> join(:array, index in fragment("indices"))
|> group_by([_i, index], index)
|> select_merge([_i, index], %{
dim0: type(fragment("?", index), :integer)
})
|> select_imported_metrics(metrics)
q
|> naive_dimension_join(events_q, metrics)
|> naive_dimension_join(pages_q, metrics)
end
def merge_imported(q, site, %Query{dimensions: [dimension]} = query, metrics)
when dimension in @imported_properties do
dim = Plausible.Stats.Filters.without_prefix(dimension)
imported_q =
site
|> Imported.Base.query_imported(query)
|> where([i], i.visitors > 0)
|> group_imported_by(dim, query)
|> select_imported_metrics(metrics)
join_on =
case dim do
_ when dim in [:url, :path] and not query.v2 ->
dynamic([s, i], s.breakdown_prop_value == i.breakdown_prop_value)
:os_version when not query.v2 ->
dynamic([s, i], s.os == i.os and s.os_version == i.os_version)
:browser_version when not query.v2 ->
dynamic([s, i], s.browser == i.browser and s.browser_version == i.browser_version)
dim ->
dynamic([s, i], field(s, ^shortname(query, dim)) == field(i, ^shortname(query, dim)))
end
from(s in Ecto.Query.subquery(q),
full_join: i in subquery(imported_q),
on: ^join_on,
select: %{}
)
|> select_joined_dimension(dim, query)
|> select_joined_metrics(metrics)
|> apply_order_by(query, metrics)
end
def merge_imported(q, site, %Query{dimensions: []} = query, metrics) do
imported_q =
site
@ -353,51 +282,82 @@ defmodule Plausible.Stats.Imported do
|> select_joined_metrics(metrics)
end
def merge_imported(q, _, _, _), do: q
def merge_imported(q, site, %Query{dimensions: ["event:goal"]} = query, metrics) do
{events, page_regexes} = Filters.Utils.split_goals_query_expressions(query.preloaded_goals)
def merge_imported_pageview_goals(q, _, %Query{include_imported: false}, _, _), do: q
Imported.Base.decide_tables(query)
|> Enum.map(fn
"imported_custom_events" ->
Imported.Base.query_imported("imported_custom_events", site, query)
|> where([i], i.visitors > 0)
|> select_merge([i], %{
dim0:
selected_as(
fragment("-indexOf(?, ?)", type(^events, {:array, :string}), i.name),
:dim0
)
})
|> select_imported_metrics(metrics)
|> group_by([], selected_as(:dim0))
|> where([], selected_as(:dim0) != 0)
def merge_imported_pageview_goals(q, site, query, page_exprs, metrics) do
if Imported.Base.decide_table(query) == "imported_pages" do
page_regexes = Enum.map(page_exprs, &Base.page_regex/1)
imported_q =
"imported_pages"
|> Imported.Base.query_imported(site, query)
"imported_pages" ->
Imported.Base.query_imported("imported_pages", site, query)
|> where([i], i.visitors > 0)
|> where(
[i],
fragment("notEmpty(multiMatchAllIndices(?, ?) as indices)", i.page, ^page_regexes)
fragment(
"notEmpty(multiMatchAllIndices(?, ?) as indices)",
i.page,
type(^page_regexes, {:array, :string})
)
)
|> join(:array, index in fragment("indices"))
|> group_by([_i, index], index)
|> select_merge([_i, index], %{
name: fragment("concat('Visit ', ?[?])", ^page_exprs, index)
dim0: selected_as(type(fragment("?", index), :integer), :dim0)
})
|> select_imported_metrics(metrics)
end)
|> Enum.reduce(q, fn imports_q, q ->
naive_dimension_join(q, imports_q, metrics)
end)
end
from(s in Ecto.Query.subquery(q),
def merge_imported(q, site, %Query{dimensions: dimensions} = query, metrics) do
if merge_imported_dimensions?(dimensions) do
imported_q =
site
|> Imported.Base.query_imported(query)
|> where([i], i.visitors > 0)
|> group_imported_by(query)
|> select_imported_metrics(metrics)
from(s in subquery(q),
full_join: i in subquery(imported_q),
on: s.name == i.name,
on: ^QueryBuilder.build_group_by_join(query),
select: %{}
)
|> select_joined_dimension(:name, query)
|> select_joined_dimensions(query)
|> select_joined_metrics(metrics)
else
q
end
end
def merge_imported(q, _, _, _), do: q
defp merge_imported_dimensions?(dimensions) do
dimensions in [["visit:browser", "visit:browser_version"], ["visit:os", "visit:os_version"]] or
(length(dimensions) == 1 and hd(dimensions) in @imported_dimensions)
end
def total_imported_visitors(site, query) do
site
|> Imported.Base.query_imported(query)
|> select_merge([i], %{total_visitors: fragment("sum(?)", i.visitors)})
end
# :TRICKY: Handle backwards compatibility with old breakdown module
defp shortname(query, _dim) when query.v2, do: :dim0
defp shortname(_query, dim), do: dim
defp select_imported_metrics(q, []), do: q
defp select_imported_metrics(q, [:visitors | rest]) do
@ -592,190 +552,175 @@ defmodule Plausible.Stats.Imported do
|> select_imported_metrics(rest)
end
defp group_imported_by(q, dim, query) when dim in [:source, :referrer] do
defp group_imported_by(q, query) do
Enum.reduce(query.dimensions, q, fn dimension, q ->
dim = Plausible.Stats.Filters.without_prefix(dimension)
group_imported_by(q, dim, shortname(query, dimension))
end)
end
defp group_imported_by(q, dim, key) when dim in [:source, :referrer] do
q
|> group_by([i], field(i, ^dim))
|> select_merge([i], %{
^shortname(query, dim) =>
fragment(
"if(empty(?), ?, ?)",
field(i, ^dim),
@no_ref,
field(i, ^dim)
^key =>
selected_as(
fragment(
"if(empty(?), ?, ?)",
field(i, ^dim),
@no_ref,
field(i, ^dim)
),
^key
)
})
end
defp group_imported_by(q, dim, query)
defp group_imported_by(q, dim, key)
when dim in [:utm_source, :utm_medium, :utm_campaign, :utm_term, :utm_content] do
q
|> group_by([i], field(i, ^dim))
|> where([i], fragment("not empty(?)", field(i, ^dim)))
|> select_merge([i], %{^shortname(query, dim) => field(i, ^dim)})
|> select_merge([i], %{^key => selected_as(field(i, ^dim), ^key)})
end
defp group_imported_by(q, :page, query) do
defp group_imported_by(q, :page, key) do
q
|> group_by([i], i.page)
|> select_merge([i], %{^shortname(query, :page) => i.page, time_on_page: sum(i.time_on_page)})
|> select_merge([i], %{^key => selected_as(i.page, ^key), time_on_page: sum(i.time_on_page)})
end
defp group_imported_by(q, :country, query) do
defp group_imported_by(q, :country, key) do
q
|> group_by([i], i.country)
|> where([i], i.country != "ZZ")
|> select_merge([i], %{^shortname(query, :country) => i.country})
|> select_merge([i], %{^key => selected_as(i.country, ^key)})
end
defp group_imported_by(q, :region, query) do
defp group_imported_by(q, :region, key) do
q
|> group_by([i], i.region)
|> where([i], i.region != "")
|> select_merge([i], %{^shortname(query, :region) => i.region})
|> select_merge([i], %{^key => selected_as(i.region, ^key)})
end
defp group_imported_by(q, :city, query) do
defp group_imported_by(q, :city, key) do
q
|> group_by([i], i.city)
|> where([i], i.city != 0 and not is_nil(i.city))
|> select_merge([i], %{^shortname(query, :city) => i.city})
|> select_merge([i], %{^key => selected_as(i.city, ^key)})
end
defp group_imported_by(q, dim, query) when dim in [:device, :browser] do
defp group_imported_by(q, dim, key) when dim in [:device, :browser] do
q
|> group_by([i], field(i, ^dim))
|> select_merge([i], %{
^shortname(query, dim) =>
fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim))
})
end
defp group_imported_by(q, :browser_version, query) do
q
|> group_by([i], [i.browser, i.browser_version])
|> select_merge([i], %{
^shortname(query, :browser) =>
fragment("if(empty(?), ?, ?)", i.browser, @not_set, i.browser),
^shortname(query, :browser_version) =>
fragment(
"if(empty(?), ?, ?)",
i.browser_version,
@not_set,
i.browser_version
^key =>
selected_as(
fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim)),
^key
)
})
end
defp group_imported_by(q, :os, query) do
defp group_imported_by(q, :browser_version, key) do
q
|> group_by([i], [i.browser_version])
|> select_merge([i], %{
^key =>
selected_as(
fragment(
"if(empty(?), ?, ?)",
i.browser_version,
@not_set,
i.browser_version
),
^key
)
})
end
defp group_imported_by(q, :os, key) do
q
|> group_by([i], i.operating_system)
|> select_merge([i], %{
^shortname(query, :os) =>
fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system)
})
end
defp group_imported_by(q, :os_version, query) do
q
|> group_by([i], [i.operating_system, i.operating_system_version])
|> select_merge([i], %{
^shortname(query, :os) =>
fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system),
^shortname(query, :os_version) =>
fragment(
"if(empty(?), ?, ?)",
i.operating_system_version,
@not_set,
i.operating_system_version
^key =>
selected_as(
fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system),
^key
)
})
end
defp group_imported_by(q, dim, query) when dim in [:entry_page, :exit_page] do
defp group_imported_by(q, :os_version, key) do
q
|> group_by([i], [i.operating_system_version])
|> select_merge([i], %{
^key =>
selected_as(
fragment(
"if(empty(?), ?, ?)",
i.operating_system_version,
@not_set,
i.operating_system_version
),
^key
)
})
end
defp group_imported_by(q, dim, key) when dim in [:entry_page, :exit_page] do
q
|> group_by([i], field(i, ^dim))
|> select_merge([i], %{^shortname(query, dim) => field(i, ^dim)})
|> select_merge([i], %{^key => selected_as(field(i, ^dim), ^key)})
end
defp group_imported_by(q, :name, query) do
defp group_imported_by(q, :name, key) do
q
|> group_by([i], i.name)
|> select_merge([i], %{^shortname(query, :name) => i.name})
|> select_merge([i], %{^key => selected_as(i.name, ^key)})
end
defp group_imported_by(q, :url, query) when query.v2 do
defp group_imported_by(q, :url, key) do
q
|> group_by([i], i.link_url)
|> select_merge([i], %{
^shortname(query, :url) => fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)
^key => selected_as(fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none), ^key)
})
end
defp group_imported_by(q, :url, _query) do
q
|> group_by([i], i.link_url)
|> select_merge([i], %{
breakdown_prop_value: fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)
})
end
defp group_imported_by(q, :path, query) when query.v2 do
defp group_imported_by(q, :path, key) do
q
|> group_by([i], i.path)
|> select_merge([i], %{
^shortname(query, :path) => fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)
^key => selected_as(fragment("if(not empty(?), ?, ?)", i.path, i.path, @none), ^key)
})
end
defp group_imported_by(q, :path, _query) do
q
|> group_by([i], i.path)
|> select_merge([i], %{
breakdown_prop_value: fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)
})
defp select_joined_dimensions(q, query) do
Enum.reduce(query.dimensions, q, fn dimension, q ->
select_joined_dimension(q, dimension, shortname(query, dimension))
end)
end
defp select_joined_dimension(q, :city, query) do
defp select_joined_dimension(q, "visit:city", key) do
select_merge(q, [s, i], %{
^shortname(query, :city) => fragment("greatest(?,?)", i.city, s.city)
^key => selected_as(fragment("greatest(?,?)", field(i, ^key), field(s, ^key)), ^key)
})
end
defp select_joined_dimension(q, :os_version, query) when not query.v2 do
defp select_joined_dimension(q, _dimension, key) do
select_merge(q, [s, i], %{
os: fragment("if(empty(?), ?, ?)", s.os, i.os, s.os),
os_version: fragment("if(empty(?), ?, ?)", s.os_version, i.os_version, s.os_version)
})
end
defp select_joined_dimension(q, :browser_version, query) when not query.v2 do
select_merge(q, [s, i], %{
browser: fragment("if(empty(?), ?, ?)", s.browser, i.browser, s.browser),
browser_version:
fragment("if(empty(?), ?, ?)", s.browser_version, i.browser_version, s.browser_version)
})
end
defp select_joined_dimension(q, dim, query) when dim in [:url, :path] and not query.v2 do
select_merge(q, [s, i], %{
breakdown_prop_value:
fragment(
"if(empty(?), ?, ?)",
s.breakdown_prop_value,
i.breakdown_prop_value,
s.breakdown_prop_value
)
})
end
defp select_joined_dimension(q, dim, query) do
select_merge(q, [s, i], %{
^shortname(query, dim) =>
fragment(
"if(empty(?), ?, ?)",
field(s, ^shortname(query, dim)),
field(i, ^shortname(query, dim)),
field(s, ^shortname(query, dim))
^key =>
selected_as(
fragment(
"if(empty(?), ?, ?)",
field(s, ^key),
field(i, ^key),
field(s, ^key)
),
^key
)
})
end
@ -882,21 +827,12 @@ defmodule Plausible.Stats.Imported do
|> select_joined_metrics(rest)
end
defp apply_order_by(q, %Query{v2: true}, _), do: q
defp apply_order_by(q, query, [:visitors | rest]) do
order_by(q, [s, i], desc: s.visitors + i.visitors)
|> apply_order_by(query, rest)
end
defp apply_order_by(q, _query, _), do: q
defp naive_dimension_join(q1, q2, metrics) do
from(a in Ecto.Query.subquery(q1),
from(a in subquery(q1),
full_join: b in subquery(q2),
on: a.dim0 == b.dim0,
select: %{
dim0: fragment("coalesce(?, ?)", a.dim0, b.dim0)
dim0: selected_as(fragment("if(? != 0, ?, ?)", a.dim0, a.dim0, b.dim0), :dim0)
}
)
|> select_joined_metrics(metrics)

7
lib/plausible/stats/metrics.ex
View File

@ -17,7 +17,8 @@ defmodule Plausible.Stats.Metrics do
:events,
:conversion_rate,
:group_conversion_rate,
:time_on_page
:time_on_page,
:percentage
] ++ on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: [])
@metric_mappings Enum.into(@all_metrics, %{}, fn metric -> {to_string(metric), metric} end)
@ -27,4 +28,8 @@ defmodule Plausible.Stats.Metrics do
def from_string!(str) do
Map.fetch!(@metric_mappings, str)
end
def from_string(str) do
Map.fetch(@metric_mappings, str)
end
end

6
lib/plausible/stats/query.ex
View File

@ -225,6 +225,12 @@ defmodule Plausible.Stats.Query do
struct!(query, filters: Filters.parse(params["filters"]))
end
def set(query, keywords) do
query
|> struct!(keywords)
|> refresh_imported_opts()
end
@spec set_dimensions(t(), list(String.t())) :: t()
def set_dimensions(query, dimensions) do
query

16
lib/plausible/stats/query_optimizer.ex
View File

@ -3,6 +3,7 @@ defmodule Plausible.Stats.QueryOptimizer do
Methods to manipulate Query for business logic reasons before building an ecto query.
"""
use Plausible
alias Plausible.Stats.{Query, TableDecider, Util}
@doc """
@ -148,8 +149,17 @@ defmodule Plausible.Stats.QueryOptimizer do
dimension -> dimension
end)
query
|> Query.set_metrics(session_metrics)
|> Query.set_dimensions(dimensions)
filters =
if "event:page" in query.dimensions do
query.filters
|> Enum.map(fn
[op, "event:page" | rest] -> [op, "visit:entry_page" | rest]
filter -> filter
end)
else
query.filters
end
Query.set(query, filters: filters, metrics: session_metrics, dimensions: dimensions)
end
end

141
lib/plausible/stats/sql/expression.ex
View File

@ -11,84 +11,121 @@ defmodule Plausible.Stats.SQL.Expression do
@no_ref "Direct / None"
@not_set "(not set)"
defmacrop field_or_blank_value(expr, empty_value) do
defmacrop field_or_blank_value(expr, empty_value, select_alias) do
quote do
dynamic(
[t],
fragment("if(empty(?), ?, ?)", unquote(expr), unquote(empty_value), unquote(expr))
selected_as(
fragment("if(empty(?), ?, ?)", unquote(expr), unquote(empty_value), unquote(expr)),
^unquote(select_alias)
)
)
end
end
def dimension("time:hour", query) do
dynamic([t], fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone))
end
def dimension("time:day", query) do
dynamic([t], fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone))
end
def dimension("time:month", query) do
dynamic([t], fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone))
end
def dimension("event:name", _query), do: dynamic([t], t.name)
def dimension("event:page", _query), do: dynamic([t], t.pathname)
def dimension("event:hostname", _query), do: dynamic([t], t.hostname)
def dimension("event:props:" <> property_name, _query) do
def dimension("time:hour", query, select_alias) do
dynamic(
[t],
fragment(
"if(not empty(?), ?, '(none)')",
get_by_key(t, :meta, ^property_name),
get_by_key(t, :meta, ^property_name)
selected_as(
fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone),
^select_alias
)
)
end
def dimension("visit:entry_page", _query), do: dynamic([t], t.entry_page)
def dimension("visit:exit_page", _query), do: dynamic([t], t.exit_page)
def dimension("time:day", query, select_alias) do
dynamic(
[t],
selected_as(
fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone),
^select_alias
)
)
end
def dimension("visit:utm_medium", _query),
do: field_or_blank_value(t.utm_medium, @not_set)
def dimension("time:month", query, select_alias) do
dynamic(
[t],
selected_as(
fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone),
^select_alias
)
)
end
def dimension("visit:utm_source", _query),
do: field_or_blank_value(t.utm_source, @not_set)
def dimension("event:name", _query, select_alias),
do: dynamic([t], selected_as(t.name, ^select_alias))
def dimension("visit:utm_campaign", _query),
do: field_or_blank_value(t.utm_campaign, @not_set)
def dimension("event:page", _query, select_alias),
do: dynamic([t], selected_as(t.pathname, ^select_alias))
def dimension("visit:utm_content", _query),
do: field_or_blank_value(t.utm_content, @not_set)
def dimension("event:hostname", _query, select_alias),
do: dynamic([t], selected_as(t.hostname, ^select_alias))
def dimension("visit:utm_term", _query),
do: field_or_blank_value(t.utm_term, @not_set)
def dimension("event:props:" <> property_name, _query, select_alias) do
dynamic(
[t],
selected_as(
fragment(
"if(not empty(?), ?, '(none)')",
get_by_key(t, :meta, ^property_name),
get_by_key(t, :meta, ^property_name)
),
^select_alias
)
)
end
def dimension("visit:source", _query),
do: field_or_blank_value(t.source, @no_ref)
def dimension("visit:entry_page", _query, select_alias),
do: dynamic([t], selected_as(t.entry_page, ^select_alias))
def dimension("visit:referrer", _query),
do: field_or_blank_value(t.referrer, @no_ref)
def dimension("visit:exit_page", _query, select_alias),
do: dynamic([t], selected_as(t.exit_page, ^select_alias))
def dimension("visit:device", _query),
do: field_or_blank_value(t.device, @not_set)
def dimension("visit:utm_medium", _query, select_alias),
do: field_or_blank_value(t.utm_medium, @not_set, select_alias)
def dimension("visit:os", _query), do: field_or_blank_value(t.os, @not_set)
def dimension("visit:utm_source", _query, select_alias),
do: field_or_blank_value(t.utm_source, @not_set, select_alias)
def dimension("visit:os_version", _query),
do: field_or_blank_value(t.os_version, @not_set)
def dimension("visit:utm_campaign", _query, select_alias),
do: field_or_blank_value(t.utm_campaign, @not_set, select_alias)
def dimension("visit:browser", _query),
do: field_or_blank_value(t.browser, @not_set)
def dimension("visit:utm_content", _query, select_alias),
do: field_or_blank_value(t.utm_content, @not_set, select_alias)
def dimension("visit:browser_version", _query),
do: field_or_blank_value(t.browser_version, @not_set)
def dimension("visit:utm_term", _query, select_alias),
do: field_or_blank_value(t.utm_term, @not_set, select_alias)
# :TODO: Locations also set extra filters
def dimension("visit:country", _query), do: dynamic([t], t.country)
def dimension("visit:region", _query), do: dynamic([t], t.region)
def dimension("visit:city", _query), do: dynamic([t], t.city)
def dimension("visit:source", _query, select_alias),
do: field_or_blank_value(t.source, @no_ref, select_alias)
def dimension("visit:referrer", _query, select_alias),
do: field_or_blank_value(t.referrer, @no_ref, select_alias)
def dimension("visit:device", _query, select_alias),
do: field_or_blank_value(t.device, @not_set, select_alias)
def dimension("visit:os", _query, select_alias),
do: field_or_blank_value(t.os, @not_set, select_alias)
def dimension("visit:os_version", _query, select_alias),
do: field_or_blank_value(t.os_version, @not_set, select_alias)
def dimension("visit:browser", _query, select_alias),
do: field_or_blank_value(t.browser, @not_set, select_alias)
def dimension("visit:browser_version", _query, select_alias),
do: field_or_blank_value(t.browser_version, @not_set, select_alias)
def dimension("visit:country", _query, select_alias),
do: dynamic([t], selected_as(t.country, ^select_alias))
def dimension("visit:region", _query, select_alias),
do: dynamic([t], selected_as(t.region, ^select_alias))
def dimension("visit:city", _query, select_alias),
do: dynamic([t], selected_as(t.city, ^select_alias))
defmacro event_goal_join(events, page_regexes) do
quote do

47
lib/plausible/stats/sql/query_builder.ex
View File

@ -7,7 +7,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
import Plausible.Stats.Imported
import Plausible.Stats.Util
alias Plausible.Stats.{Base, Query, QueryOptimizer, TableDecider, Filters, Metrics}
alias Plausible.Stats.{Base, Query, QueryOptimizer, TableDecider, Filters}
alias Plausible.Stats.SQL.Expression
require Plausible.Stats.SQL.Expression
@ -44,6 +44,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
|> merge_imported(site, events_query, events_query.metrics)
|> maybe_add_global_conversion_rate(site, events_query)
|> maybe_add_group_conversion_rate(site, events_query)
|> Base.add_percentage_metric(site, events_query, events_query.metrics)
end
defp join_sessions_if_needed(q, site, query) do
@ -84,6 +85,9 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
|> join_events_if_needed(site, sessions_query)
|> build_group_by(sessions_query)
|> merge_imported(site, sessions_query, sessions_query.metrics)
|> maybe_add_global_conversion_rate(site, sessions_query)
|> maybe_add_group_conversion_rate(site, sessions_query)
|> Base.add_percentage_metric(site, sessions_query, sessions_query.metrics)
end
def join_events_if_needed(q, site, query) do
@ -123,36 +127,23 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
^shortname(query, dimension) => fragment("?", goal)
},
group_by: goal,
where: goal != 0
where: goal != 0 and (e.name == "pageview" or goal < 0)
)
end
defp dimension_group_by(q, query, dimension) do
key = shortname(query, dimension)
q
|> select_merge(^%{shortname(query, dimension) => Expression.dimension(dimension, query)})
|> group_by(^Expression.dimension(dimension, query))
|> select_merge(^%{key => Expression.dimension(dimension, query, key)})
|> group_by([], selected_as(^key))
end
defp build_order_by(q, query, mode) do
Enum.reduce(query.order_by || [], q, &build_order_by(&2, query, &1, mode))
defp build_order_by(q, query) do
Enum.reduce(query.order_by || [], q, &build_order_by(&2, query, &1))
end
def build_order_by(q, query, {metric_or_dimension, order_direction}, :inner) do
order_by(
q,
[t],
^{
order_direction,
if(
Metrics.metric?(metric_or_dimension),
do: dynamic([], selected_as(^shortname(query, metric_or_dimension))),
else: Expression.dimension(metric_or_dimension, query)
)
}
)
end
def build_order_by(q, query, {metric_or_dimension, order_direction}, :outer) do
def build_order_by(q, query, {metric_or_dimension, order_direction}) do
order_by(
q,
[t],
@ -262,10 +253,10 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
defp join_query_results({nil, _}, {nil, _}), do: nil
defp join_query_results({events_q, events_query}, {nil, _}),
do: events_q |> build_order_by(events_query, :inner)
do: events_q |> build_order_by(events_query)
defp join_query_results({nil, _}, {sessions_q, sessions_query}),
do: sessions_q |> build_order_by(sessions_query, :inner)
defp join_query_results({nil, events_query}, {sessions_q, _}),
do: sessions_q |> build_order_by(events_query)
defp join_query_results({events_q, events_query}, {sessions_q, sessions_query}) do
join(subquery(events_q), :left, [e], s in subquery(sessions_q),
@ -274,12 +265,12 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
|> select_join_fields(events_query, events_query.dimensions, e)
|> select_join_fields(events_query, events_query.metrics, e)
|> select_join_fields(sessions_query, List.delete(sessions_query.metrics, :sample_percent), s)
|> build_order_by(events_query, :outer)
|> build_order_by(events_query)
end
defp build_group_by_join(%Query{dimensions: []}), do: true
def build_group_by_join(%Query{dimensions: []}), do: true
defp build_group_by_join(query) do
def build_group_by_join(query) do
query.dimensions
|> Enum.map(fn dim ->
dynamic([e, s], field(e, ^shortname(query, dim)) == field(s, ^shortname(query, dim)))

28
lib/plausible/stats/table_decider.ex
View File

@ -29,34 +29,42 @@ defmodule Plausible.Stats.TableDecider do
|> filter_keys()
|> partition(query, &filters_partitioner/2)
%{event: event_only_dimensions, session: session_only_dimensions} =
partition(query.dimensions, query, &filters_partitioner/2)
cond do
# Only one table needs to be queried
empty?(event_only_metrics) && empty?(event_only_filters) ->
empty?(event_only_metrics) && empty?(event_only_filters) && empty?(event_only_dimensions) ->
{[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}
empty?(session_only_metrics) && empty?(session_only_filters) ->
empty?(session_only_metrics) && empty?(session_only_filters) &&
empty?(session_only_dimensions) ->
{event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics}
# Filters on both events and sessions, but only one kind of metric
empty?(event_only_metrics) ->
# Filters and/or dimensions on both events and sessions, but only one kind of metric
empty?(event_only_metrics) && empty?(event_only_dimensions) ->
{[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}
empty?(session_only_metrics) ->
empty?(session_only_metrics) && empty?(session_only_dimensions) ->
{event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics}
# Default: prefer sessions
# Default: prefer events
true ->
{event_only_metrics ++ sample_percent,
session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}
{event_only_metrics ++ either_metrics ++ sample_percent,
session_only_metrics ++ sample_percent, other_metrics}
end
end
defp filter_keys(query) do
query.filters
|> Enum.map(fn [_, filter_key | _rest] -> filter_key end)
|> Enum.concat(query.dimensions)
end
defp metric_partitioner(%Query{v2: true}, :conversion_rate), do: :either
defp metric_partitioner(%Query{v2: true}, :group_conversion_rate), do: :either
defp metric_partitioner(%Query{v2: true}, :visitors), do: :either
defp metric_partitioner(%Query{v2: true}, :visits), do: :either
defp metric_partitioner(_, :conversion_rate), do: :event
defp metric_partitioner(_, :group_conversion_rate), do: :event
defp metric_partitioner(_, :average_revenue), do: :event
@ -76,7 +84,7 @@ defmodule Plausible.Stats.TableDecider do
# Calculated metrics - handled on callsite separately from other metrics.
defp metric_partitioner(_, :time_on_page), do: :other
defp metric_partitioner(_, :total_visitors), do: :other
defp metric_partitioner(_, :percentage), do: :other
defp metric_partitioner(_, :percentage), do: :either
# Sample percentage is included in both tables if queried.
defp metric_partitioner(_, :sample_percent), do: :sample_percent

8
test/plausible/imported/csv_importer_test.exs
View File

@ -622,7 +622,7 @@ defmodule Plausible.Imported.CSVImporterTest do
case params_or_site do
%Plausible.Site{} = site ->
common_params.(site)
|> Map.put("metrics", "visitors,visits,pageviews,visit_duration,bounce_rate")
|> Map.put("metrics", "visitors,visits,visit_duration,bounce_rate")
|> Map.put("limit", 1000)
|> Map.put("property", by)
@ -669,7 +669,7 @@ defmodule Plausible.Imported.CSVImporterTest do
assert exported["pageviews"] == imported["pageviews"]
assert exported["bounce_rate"] == imported["bounce_rate"]
assert_in_delta exported["visitors"], imported["visitors"], 1
assert exported["visits"] == imported["visits"]
assert_in_delta exported["visits"], imported["visits"], 1
assert_in_delta exported["visit_duration"], imported["visit_duration"], 1
end)
@ -810,7 +810,7 @@ defmodule Plausible.Imported.CSVImporterTest do
_no_diff = 0
end
end)
) == [0.0, 0.0, 0.0, 0.0, 0.03614457831325302]
) == [0.0, 0.0, 0.0, 0.0, 0.0]
# NOTE: city breakdown's visit duration difference is up to 14%
assert summary(field(exported_cities, "visit_duration")) == [0, 0.0, 0.0, 1.0, 1718]
@ -829,7 +829,7 @@ defmodule Plausible.Imported.CSVImporterTest do
_no_diff = 0
end
end)
) == [0, 0.0, 0.0, 0.0, 0.1428571428571429]
) == [0, 0.0, 0.0, 0.0, 0.0]
# NOTE: city breakdown's visitors relative difference is up to 27%
assert summary(field(exported_cities, "visitors")) == [1, 1.0, 1.0, 2.0, 22]

60
test/plausible/imported/google_analytics4_test.exs
View File

@ -117,7 +117,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
breakdown_params =
common_params
|> Map.put("metrics", "visitors,visits,pageviews,visit_duration,bounce_rate")
|> Map.put("metrics", "visitors,visits,visit_duration,bounce_rate")
|> Map.put("limit", 1000)
%{key: api_key} = insert(:api_key, user: user)
@ -467,7 +467,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.first(results) == %{
"bounce_rate" => 35.0,
"pageviews" => 6229,
"visit_duration" => 40.0,
"visitors" => 4671,
"visits" => 4917,
@ -475,12 +474,11 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
}
assert List.last(results) == %{
"bounce_rate" => 100.0,
"pageviews" => 1,
"visit_duration" => 0.0,
"bounce_rate" => 0.0,
"source" => "yahoo",
"visit_duration" => 41.0,
"visitors" => 1,
"visits" => 1,
"source" => "petalsearch.com"
"visits" => 1
}
end
@ -493,7 +491,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert [
%{
"bounce_rate" => 35.0,
"pageviews" => 6399,
"utm_medium" => "organic",
"visit_duration" => 40.0,
"visitors" => 4787,
@ -501,7 +498,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
},
%{
"bounce_rate" => 58.0,
"pageviews" => 491,
"utm_medium" => "referral",
"visit_duration" => 27.0,
"visitors" => 294,
@ -520,7 +516,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.first(results) == %{
"bounce_rate" => 35.0,
"pageviews" => 838,
"visit_duration" => 43.0,
"visitors" => 675,
"visits" => 712,
@ -529,11 +524,10 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.last(results) == %{
"bounce_rate" => 0.0,
"pageviews" => 1,
"visit_duration" => 27.0,
"entry_page" => "/znamenitosti-rima-koje-treba-vidjeti",
"visit_duration" => 40.0,
"visitors" => 1,
"visits" => 1,
"entry_page" => "/kad-lisce-pada"
"visits" => 1
}
end
@ -543,12 +537,11 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
%{"results" => results} =
get(conn, "/api/v1/stats/breakdown", params) |> json_response(200)
assert length(results) == 488
assert length(results) == 494
assert List.first(results) == %{
"bounce_rate" => 35.0,
"city" => 792_680,
"pageviews" => 1650,
"visit_duration" => 39.0,
"visitors" => 1233,
"visits" => 1273
@ -556,9 +549,8 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.last(results) == %{
"bounce_rate" => 0.0,
"city" => 4_399_605,
"pageviews" => 7,
"visit_duration" => 128.0,
"city" => 11_951_298,
"visit_duration" => 271.0,
"visitors" => 1,
"visits" => 1
}
@ -574,7 +566,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.first(results) == %{
"bounce_rate" => 38.0,
"pageviews" => 7041,
"visit_duration" => 37.0,
"visitors" => 5277,
"visits" => 5532,
@ -583,7 +574,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.last(results) == %{
"bounce_rate" => 37.0,
"pageviews" => 143,
"visit_duration" => 60.0,
"visitors" => 97,
"visits" => 100,
@ -601,7 +591,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.first(results) == %{
"bounce_rate" => 33.0,
"pageviews" => 8143,
"visit_duration" => 50.0,
"visitors" => 4625,
"visits" => 4655,
@ -610,7 +599,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.last(results) == %{
"bounce_rate" => 0.0,
"pageviews" => 6,
"visit_duration" => 0.0,
"visitors" => 1,
"visits" => 1,
@ -628,7 +616,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.first(results) == %{
"bounce_rate" => 34.0,
"pageviews" => 5827,
"visit_duration" => 41.0,
"visitors" => 4319,
"visits" => 4495,
@ -637,7 +624,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.last(results) == %{
"bounce_rate" => 0.0,
"pageviews" => 6,
"visit_duration" => 0.0,
"visitors" => 1,
"visits" => 1,
@ -657,7 +643,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
"bounce_rate" => 32.0,
"os" => "Android",
"os_version" => "13.0.0",
"pageviews" => 1673,
"visit_duration" => 42.0,
"visitors" => 1247,
"visits" => 1295
@ -665,17 +650,16 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.last(results) == %{
"bounce_rate" => 0.0,
"os" => "iOS",
"os_version" => "15.1",
"pageviews" => 1,
"visit_duration" => 54.0,
"os" => "Chrome OS",
"os_version" => "x86_64 15662.76.0",
"visit_duration" => 16.0,
"visitors" => 1,
"visits" => 1
}
end
defp assert_pages(conn, params) do
metrics = "visitors,visits,pageviews,time_on_page,visit_duration,bounce_rate"
metrics = "visitors,visits,time_on_page,visit_duration,bounce_rate"
params =
params
@ -686,7 +670,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
%{"results" => results} =
get(conn, "/api/v1/stats/breakdown", params) |> json_response(200)
assert length(results) == 729
assert length(results) == 730
# The `event:page` breakdown is currently using the `entry_page`
# property to allow querying session metrics.
@ -696,7 +680,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
# it will allow us to assert on the session metrics as well.
assert Enum.at(results, 2) == %{
"page" => "/",
"pageviews" => 5537,
"time_on_page" => 17.677262055264585,
"visitors" => 371,
"visits" => 212,
@ -707,13 +690,12 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
# This page was never an entry_page in the imported data, and
# therefore the session metrics are returned as `nil`.
assert List.last(results) == %{
"page" => "/5-dobrih-razloga-zasto-zapoceti-dan-zobenom-kasom/",
"pageviews" => 2,
"time_on_page" => 10.0,
"bounce_rate" => 0.0,
"page" => "/znamenitosti-rima-koje-treba-vidjeti/",
"time_on_page" => 40.0,
"visit_duration" => 0.0,
"visitors" => 1,
"visits" => 1,
"bounce_rate" => nil,
"visit_duration" => nil
"visits" => 1
}
end
end

6
test/plausible/stats/table_decider_test.exs
View File

@ -76,7 +76,7 @@ defmodule Plausible.Stats.TableDeciderTest do
query = make_query(false, [])
assert partition_metrics([:time_on_page, :percentage, :total_visitors], query) ==
{[], [], [:time_on_page, :percentage, :total_visitors]}
{[], [:percentage], [:time_on_page, :total_visitors]}
end
test "raises if unknown metric" do
@ -108,11 +108,11 @@ defmodule Plausible.Stats.TableDeciderTest do
{[], [:visit_duration, :visits], []}
end
test "metrics that can be calculated on either are biased to sessions" do
test "metrics that can be calculated on either are biased to events" do
query = make_query(true, [])
assert partition_metrics([:bounce_rate, :total_revenue, :visitors], query) ==
{[:total_revenue], [:bounce_rate, :visitors], []}
{[:total_revenue, :visitors], [:bounce_rate], []}
end
test "sample_percent is handled with either metrics" do

2
test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv
View File

@ -1,2 +1,2 @@
name,visitors,pageviews,bounce_rate,time_on_page
/some-other-page,1,1,,60.0
/some-other-page,1,1,0,60.0

1 name visitors pageviews bounce_rate time_on_page
2 /some-other-page 1 1 0 60.0

2
test/plausible_web/controllers/CSVs/30d/pages.csv
View File

@ -1,4 +1,4 @@
name,visitors,pageviews,bounce_rate,time_on_page
/,4,3,67,
/signup,1,1,0,60.0
/some-other-page,1,1,,60.0
/some-other-page,1,1,0,60.0

1 name visitors pageviews bounce_rate time_on_page
2 / 4 3 67
3 /signup 1 1 0 60.0
4 /some-other-page 1 1 0 60.0

2
test/plausible_web/controllers/CSVs/6m/pages.csv
View File

@ -1,4 +1,4 @@
name,visitors,pageviews,bounce_rate,time_on_page
/,5,4,75,
/signup,1,1,0,60.0
/some-other-page,1,1,,60.0
/some-other-page,1,1,0,60.0

1 name visitors pageviews bounce_rate time_on_page
2 / 5 4 75
3 /signup 1 1 0 60.0
4 /some-other-page 1 1 0 60.0

33
test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
View File

@ -890,8 +890,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
assert json_response(conn, 200) == %{
"results" => [
%{"page" => "/", "pageviews" => 2},
%{"page" => "/plausible.io", "pageviews" => 1},
%{"page" => "/include-me", "pageviews" => 1}
%{"page" => "/include-me", "pageviews" => 1},
%{"page" => "/plausible.io", "pageviews" => 1}
]
}
end
@ -1023,7 +1023,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
"period" => "day",
"date" => "2021-01-01",
"property" => "visit:exit_page",
"metrics" => "visitors,visits,pageviews,bounce_rate,visit_duration,events",
"metrics" => "visitors,visits,bounce_rate,visit_duration",
"with_imported" => "true"
})
@ -1031,18 +1031,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
"results" => [
%{
"bounce_rate" => 0.0,
"events" => 7,
"exit_page" => "/b",
"pageviews" => 7,
"visit_duration" => 150.0,
"visitors" => 3,
"visits" => 4
},
%{
"bounce_rate" => 100.0,
"events" => 1,
"exit_page" => "/a",
"pageviews" => 1,
"visit_duration" => 0.0,
"visitors" => 1,
"visits" => 1
@ -2176,8 +2172,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
assert json_response(conn, 200) == %{
"results" => [
%{"page" => "/plausible.io", "bounce_rate" => 100},
%{"page" => "/important-page", "bounce_rate" => 100}
%{"page" => "/important-page", "bounce_rate" => 100},
%{"page" => "/plausible.io", "bounce_rate" => 100}
]
}
end
@ -2596,14 +2592,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
assert json_response(conn, 200) == %{
"results" => [
%{
"page" => "/B",
"time_on_page" => 90.0
},
%{
"page" => "/A",
"time_on_page" => 60.0
},
%{
"page" => "/B",
"time_on_page" => 90.0
},
%{
"page" => "/C",
"time_on_page" => nil
@ -3045,13 +3041,13 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
assert json_response(conn, 200) == %{
"results" => [
%{
"entry_page" => "/entry-page-1",
"bounce_rate" => 0
},
%{
"entry_page" => "/entry-page-2",
"bounce_rate" => 100
},
%{
"entry_page" => "/entry-page-1",
"bounce_rate" => 0
}
]
}
@ -3146,6 +3142,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
},
%{
"page" => "/plausible.io",
# Breaks for event:page breakdown since visitors is calculated based on entry_page :/
"visitors" => 2,
"bounce_rate" => 100,
"visit_duration" => 0,
@ -3290,8 +3287,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
assert %{"browser" => "Chrome", "events" => 1} = breakdown_and_first.("visit:browser")
assert %{"device" => "Desktop", "events" => 1} = breakdown_and_first.("visit:device")
assert %{"entry_page" => "/test", "events" => 1} = breakdown_and_first.("visit:entry_page")
assert %{"exit_page" => "/test", "events" => 1} = breakdown_and_first.("visit:exit_page")
assert %{"country" => "EE", "events" => 1} = breakdown_and_first.("visit:country")
assert %{"os" => "Mac", "events" => 1} = breakdown_and_first.("visit:os")
assert %{"page" => "/test", "events" => 1} = breakdown_and_first.("event:page")

104
test/plausible_web/controllers/api/external_stats_controller/query_test.exs
View File

@ -1477,7 +1477,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visitors"],
"metrics" => ["visitors", "percentage"],
"date_range" => ["2021-01-01", "2021-01-01"],
"dimensions" => [unquote(dimension)]
})
@ -1485,9 +1485,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
%{"results" => results} = json_response(conn, 200)
assert results == [
%{"dimensions" => [unquote(value1)], "metrics" => [3]},
%{"dimensions" => [unquote(value2)], "metrics" => [2]},
%{"dimensions" => [unquote(blank_value)], "metrics" => [1]}
%{"dimensions" => [unquote(value1)], "metrics" => [3, 50]},
%{"dimensions" => [unquote(value2)], "metrics" => [2, 33.3]},
%{"dimensions" => [unquote(blank_value)], "metrics" => [1, 16.7]}
]
end
end
@ -3463,6 +3463,48 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
%{"dimensions" => ["Chrome"], "metrics" => [1]}
]
end
test "all metrics for breakdown by event prop", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
user_id: 1,
pathname: "/",
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: 1,
pathname: "/plausible.io",
timestamp: ~N[2021-01-01 00:10:00]
),
build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview,
pathname: "/plausible.io",
timestamp: ~N[2021-01-01 00:00:00]
)
])
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => [
"visitors",
"visits",
"pageviews",
"events",
"bounce_rate",
"visit_duration"
],
"date_range" => "all",
"dimensions" => ["event:page"]
})
%{"results" => results} = json_response(conn, 200)
assert results == [
%{"dimensions" => ["/"], "metrics" => [2, 2, 2, 2, 50, 300]},
%{"dimensions" => ["/plausible.io"], "metrics" => [2, 2, 2, 2, 100, 0]}
]
end
end
describe "imported data" do
@ -3589,10 +3631,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
assert %{"dimensions" => ["Desktop"], "metrics" => [1]} =
breakdown_and_first.("visit:device")
# :TODO: These should not pass validation - not available on events.
# visit dimension and event-only metric
# assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("visit:entry_page")
# assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("visit:exit_page")
assert %{"dimensions" => ["EE"], "metrics" => [1]} = breakdown_and_first.("visit:country")
assert %{"dimensions" => ["Mac"], "metrics" => [1]} = breakdown_and_first.("visit:os")
assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("event:page")
@ -3865,6 +3903,56 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
assert meta["warning"] =~ "Imported stats are not included in the results"
end
test "imported country, region and city data",
%{
conn: conn,
site: site
} do
site_import = insert(:site_import, site: site)
populate_stats(site, site_import.id, [
build(:pageview,
timestamp: ~N[2021-01-01 00:15:00],
country_code: "DE",
subdivision1_code: "DE-BE",
city_geoname_id: 2_950_159
),
build(:pageview,
timestamp: ~N[2021-01-01 00:15:00],
country_code: "DE",
subdivision1_code: "DE-BE",
city_geoname_id: 2_950_159
),
build(:pageview,
timestamp: ~N[2021-01-01 00:15:00],
country_code: "EE",
subdivision1_code: "EE-37",
city_geoname_id: 588_409
),
build(:imported_locations, country: "EE", region: "EE-37", city: 588_409, visitors: 33)
])
for {dimension, stats_value, imports_value} <- [
{"visit:country", "DE", "EE"},
{"visit:region", "DE-BE", "EE-37"},
{"visit:city", 2_950_159, 588_409}
] do
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visitors"],
"date_range" => "all",
"dimensions" => [dimension],
"include" => %{"imports" => true}
})
assert json_response(conn, 200)["results"] == [
%{"dimensions" => [imports_value], "metrics" => [34]},
%{"dimensions" => [stats_value], "metrics" => [2]}
]
end
end
end
test "multiple breakdown timeseries with sources", %{conn: conn, site: site} do

1
test/plausible_web/controllers/api/stats_controller/conversions_test.exs
View File

@ -207,6 +207,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
]
end
@tag capture_log: true
test "garbage filters don't crash the call", %{conn: conn, site: site} do
filters =
"{\"source\":\"Direct / None\",\"screen\":\"Desktop\",\"browser\":\"Chrome\",\"os\":\"Mac\",\"os_version\":\"10.15\",\"country\":\"DE\",\"city\":\"2950159\"}%' AND 2*3*8=6*8 AND 'L9sv'!='L9sv%"

14
test/plausible_web/controllers/api/stats_controller/imported_test.exs
View File

@ -601,17 +601,17 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
)
assert json_response(conn, 200)["results"] == [
%{
"name" => "blog",
"visitors" => 2,
"bounce_rate" => 50.0,
"visit_duration" => 50.0
},
%{
"name" => "ad",
"visitors" => 2,
"bounce_rate" => 100.0,
"visit_duration" => 50.0
},
%{
"name" => "blog",
"visitors" => 2,
"bounce_rate" => 50.0,
"visit_duration" => 50.0
}
]
end
@ -708,7 +708,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
assert json_response(conn, 200)["results"] == [
%{
"bounce_rate" => nil,
"bounce_rate" => 0,
"time_on_page" => 60,
"visitors" => 3,
"pageviews" => 4,

16
test/plausible_web/controllers/api/stats_controller/pages_test.exs
View File

@ -340,7 +340,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"name" => "/blog/other-post",
"visitors" => 1,
"pageviews" => 1,
"bounce_rate" => nil,
"bounce_rate" => 0,
"time_on_page" => nil
}
]
@ -392,7 +392,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"name" => "/blog/other-post",
"visitors" => 1,
"pageviews" => 1,
"bounce_rate" => nil,
"bounce_rate" => 0,
"time_on_page" => nil
}
]
@ -744,7 +744,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"name" => "/blog/post-2",
"visitors" => 1,
"pageviews" => 1,
"bounce_rate" => nil,
"bounce_rate" => 0,
"time_on_page" => nil
}
]
@ -789,7 +789,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"name" => "/blog/(/post-2",
"visitors" => 1,
"pageviews" => 1,
"bounce_rate" => nil,
"bounce_rate" => 0,
"time_on_page" => nil
}
]
@ -842,7 +842,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"name" => "/about",
"visitors" => 1,
"pageviews" => 1,
"bounce_rate" => nil,
"bounce_rate" => 0,
"time_on_page" => nil
}
]
@ -940,7 +940,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"name" => "/"
},
%{
"bounce_rate" => nil,
"bounce_rate" => 0,
"time_on_page" => nil,
"visitors" => 1,
"pageviews" => 1,
@ -1066,7 +1066,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"visitors" => 2
},
%{
"bounce_rate" => nil,
"bounce_rate" => 0,
"name" => "/exit-blog",
"pageviews" => 1,
"time_on_page" => nil,
@ -1192,7 +1192,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"name" => "/"
},
%{
"bounce_rate" => nil,
"bounce_rate" => 0,
"time_on_page" => 60,
"visitors" => 2,
"pageviews" => 2,

29
test/plausible_web/controllers/api/stats_controller/sources_test.exs
View File

@ -453,6 +453,9 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
referrer_source: "DuckDuckGo",
referrer: "duckduckgo.com"
),
build(:imported_sources,
source: "DuckDuckGo"
),
build(:imported_sources,
source: "DuckDuckGo"
)
@ -467,7 +470,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
conn = get(conn, "/api/stats/#{site.domain}/sources?limit=1&page=2&with_imported=true")
assert json_response(conn, 200)["results"] == [
%{"name" => "DuckDuckGo", "visitors" => 2}
%{"name" => "Google", "visitors" => 2}
]
end
@ -590,17 +593,17 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
)
assert json_response(conn, 200)["results"] == [
%{
"name" => "social",
"visitors" => 1,
"bounce_rate" => 0,
"visit_duration" => 900
},
%{
"name" => "email",
"visitors" => 1,
"bounce_rate" => 100,
"visit_duration" => 0
},
%{
"name" => "social",
"visitors" => 1,
"bounce_rate" => 0,
"visit_duration" => 900
}
]
@ -611,17 +614,17 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
)
assert json_response(conn, 200)["results"] == [
%{
"name" => "social",
"visitors" => 2,
"bounce_rate" => 50,
"visit_duration" => 800.0
},
%{
"name" => "email",
"visitors" => 2,
"bounce_rate" => 50,
"visit_duration" => 50
},
%{
"name" => "social",
"visitors" => 2,
"bounce_rate" => 50,
"visit_duration" => 800.0
}
]
end