plausible/analytics
1
1
Fork 0
mirror of https://github.com/plausible/analytics.git synced 2024-09-11 18:07:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

Revert "Move conversion_rate logic from elixir to clickhouse (#3887)" (#3923)

Browse Source 
This reverts commit 1909743b90.
This commit is contained in:
Karl-Aksel Puulmann 2024-03-21 09:53:31 +02:00 committed by GitHub
parent 1909743b90
commit 253fb5d67d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 346 additions and 348 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
extra/lib/plausible/stats/goal/revenue.ex
View File

@ -10,17 +10,21 @@ defmodule Plausible.Stats.Goal.Revenue do
@revenue_metrics
end
def total_revenue_query() do
dynamic(
[e],
fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)
def total_revenue_query(query) do
from(e in query,
select_merge: %{
total_revenue:
fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)
}
)
end
def average_revenue_query() do
dynamic(
[e],
fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)
def average_revenue_query(query) do
from(e in query,
select_merge: %{
average_revenue:
fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)
}
)
end

36
lib/plausible/stats/aggregate.ex
View File

@ -11,9 +11,7 @@ defmodule Plausible.Stats.Aggregate do
:visitors,
:pageviews,
:events,
:sample_percent,
:conversion_rate,
:total_visitors
:sample_percent
] ++ @revenue_metrics
@session_metrics [:visits, :bounce_rate, :visit_duration, :views_per_visit, :sample_percent]
@ -47,6 +45,7 @@ defmodule Plausible.Stats.Aggregate do
Plausible.ClickhouseRepo.parallel_tasks([session_task, event_task, time_on_page_task])
|> Enum.reduce(%{}, fn aggregate, task_result -> Map.merge(aggregate, task_result) end)
|> maybe_put_cr(site, query, metrics)
|> Util.keep_requested_metrics(metrics)
|> cast_revenue_metrics_to_money(currency)
|> Enum.map(&maybe_round_value/1)
@ -54,20 +53,45 @@ defmodule Plausible.Stats.Aggregate do
|> Enum.into(%{})
end
defp maybe_put_cr(aggregate_result, site, query, metrics) do
if :conversion_rate in metrics do
all =
query
|> Query.remove_event_filters([:goal, :props])
|> then(fn query -> aggregate_events(site, query, [:visitors]) end)
|> Map.fetch!(:visitors)
converted = aggregate_result.visitors
cr = Util.calculate_cr(all, converted)
aggregate_result = Map.put(aggregate_result, :conversion_rate, cr)
if :total_visitors in metrics do
Map.put(aggregate_result, :total_visitors, all)
else
aggregate_result
end
else
aggregate_result
end
end
defp aggregate_events(_, _, []), do: %{}
defp aggregate_events(site, query, metrics) do
from(e in base_event_query(site, query), select: ^select_event_metrics(metrics))
from(e in base_event_query(site, query), select: %{})
|> select_event_metrics(metrics)
|> merge_imported(site, query, :aggregate, metrics)
|> maybe_add_conversion_rate(site, query, metrics, include_imported: query.include_imported)
|> ClickhouseRepo.one()
end
defp aggregate_sessions(_, _, []), do: %{}
defp aggregate_sessions(site, query, metrics) do
from(e in query_sessions(site, query), select: ^select_session_metrics(metrics, query))
from(e in query_sessions(site, query), select: %{})
|> filter_converted_sessions(site, query)
|> select_session_metrics(metrics, query)
|> merge_imported(site, query, :aggregate, metrics)
|> ClickhouseRepo.one()
|> Util.keep_requested_metrics(metrics)

253
lib/plausible/stats/base.ex
View File

@ -209,152 +209,163 @@ defmodule Plausible.Stats.Base do
def apply_entry_prop_filter(sessions_q, _, _), do: sessions_q
def select_event_metrics(metrics) do
metrics
|> Enum.map(&select_event_metric/1)
|> Enum.reduce(%{}, &Map.merge/2)
end
def select_event_metrics(q, []), do: q
defp select_event_metric(:pageviews) do
%{
pageviews:
dynamic(
[e],
def select_event_metrics(q, [:pageviews | rest]) do
from(e in q,
select_merge: %{
pageviews:
fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name)
)
}
}
)
|> select_event_metrics(rest)
end
defp select_event_metric(:events) do
%{
events: dynamic([], fragment("toUInt64(round(count(*) * any(_sample_factor)))"))
}
def select_event_metrics(q, [:events | rest]) do
from(e in q,
select_merge: %{events: fragment("toUInt64(round(count(*) * any(_sample_factor)))")}
)
|> select_event_metrics(rest)
end
defp select_event_metric(:visitors) do
%{
visitors: dynamic([e], selected_as(fragment(@uniq_users_expression, e.user_id), :visitors))
}
def select_event_metrics(q, [:visitors | rest]) do
from(e in q,
select_merge: %{
visitors: selected_as(fragment(@uniq_users_expression, e.user_id), :visitors)
}
)
|> select_event_metrics(rest)
end
on_full_build do
defp select_event_metric(:total_revenue) do
%{total_revenue: Plausible.Stats.Goal.Revenue.total_revenue_query()}
def select_event_metrics(q, [:total_revenue | rest]) do
q
|> Plausible.Stats.Goal.Revenue.total_revenue_query()
|> select_event_metrics(rest)
end
defp select_event_metric(:average_revenue) do
%{average_revenue: Plausible.Stats.Goal.Revenue.average_revenue_query()}
def select_event_metrics(q, [:average_revenue | rest]) do
q
|> Plausible.Stats.Goal.Revenue.average_revenue_query()
|> select_event_metrics(rest)
end
end
defp select_event_metric(:sample_percent) do
%{
sample_percent:
dynamic(
[],
def select_event_metrics(q, [:sample_percent | rest]) do
from(e in q,
select_merge: %{
sample_percent:
fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)")
)
}
}
)
|> select_event_metrics(rest)
end
defp select_event_metric(:percentage), do: %{}
defp select_event_metric(:conversion_rate), do: %{}
defp select_event_metric(:total_visitors), do: %{}
defp select_event_metric(unknown), do: raise("Unknown metric: #{unknown}")
def select_session_metrics(metrics, query) do
metrics
|> Enum.map(&select_session_metric(&1, query))
|> Enum.reduce(%{}, &Map.merge/2)
def select_event_metrics(q, [:percentage | rest]) do
q |> select_event_metrics(rest)
end
defp select_session_metric(:bounce_rate, query) do
def select_event_metrics(_, [unknown | _]), do: raise("Unknown metric: #{unknown}")
def select_session_metrics(q, [], _query), do: q
def select_session_metrics(q, [:bounce_rate | rest], query) do
condition = dynamic_filter_condition(query, "event:page", :entry_page)
%{
bounce_rate:
dynamic(
[],
fragment(
"toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))",
^condition,
^condition
)
),
__internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))
}
from(s in q,
select_merge:
^%{
bounce_rate:
dynamic(
[],
fragment(
"toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))",
^condition,
^condition
)
),
__internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))
}
)
|> select_session_metrics(rest, query)
end
defp select_session_metric(:visits, _query) do
%{
visits: dynamic([s], fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign))
}
def select_session_metrics(q, [:visits | rest], query) do
from(s in q,
select_merge: %{
visits: fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign)
}
)
|> select_session_metrics(rest, query)
end
defp select_session_metric(:pageviews, _query) do
%{
pageviews:
dynamic(
[s],
def select_session_metrics(q, [:pageviews | rest], query) do
from(s in q,
select_merge: %{
pageviews:
fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews)
)
}
}
)
|> select_session_metrics(rest, query)
end
defp select_session_metric(:events, _query) do
%{
events:
dynamic(
[s],
fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events)
)
}
def select_session_metrics(q, [:events | rest], query) do
from(s in q,
select_merge: %{
events: fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events)
}
)
|> select_session_metrics(rest, query)
end
defp select_session_metric(:visitors, _query) do
%{
visitors:
dynamic(
[s],
def select_session_metrics(q, [:visitors | rest], query) do
from(s in q,
select_merge: %{
visitors:
selected_as(
fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", s.user_id),
:visitors
)
)
}
}
)
|> select_session_metrics(rest, query)
end
defp select_session_metric(:visit_duration, _query) do
%{
visit_duration:
dynamic([], fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))")),
__internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))
}
def select_session_metrics(q, [:visit_duration | rest], query) do
from(s in q,
select_merge: %{
:visit_duration =>
fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))"),
__internal_visits: fragment("toUInt32(sum(sign))")
}
)
|> select_session_metrics(rest, query)
end
defp select_session_metric(:views_per_visit, _query) do
%{
views_per_visit:
dynamic(
[s],
fragment("ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)", s.sign, s.pageviews, s.sign)
),
__internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))
}
def select_session_metrics(q, [:views_per_visit | rest], query) do
from(s in q,
select_merge: %{
views_per_visit:
fragment("ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)", s.sign, s.pageviews, s.sign),
__internal_visits: fragment("toUInt32(sum(sign))")
}
)
|> select_session_metrics(rest, query)
end
defp select_session_metric(:sample_percent, _query) do
%{
sample_percent:
dynamic(
[],
def select_session_metrics(q, [:sample_percent | rest], query) do
from(e in q,
select_merge: %{
sample_percent:
fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)")
)
}
}
)
|> select_session_metrics(rest, query)
end
defp select_session_metric(:percentage, _query), do: %{}
def select_session_metrics(q, [:percentage | rest], query) do
q |> select_session_metrics(rest, query)
end
def dynamic_filter_condition(query, filter_key, db_field) do
case query && query.filters && query.filters[filter_key] do
@ -575,7 +586,7 @@ defmodule Plausible.Stats.Base do
|> select([e], total_visitors: fragment(@uniq_users_expression, e.user_id))
end
defp total_visitors_subquery(site, query, true) do
defp total_visitors_subquery(site, %Query{include_imported: true} = query) do
dynamic(
[e],
selected_as(
@ -586,16 +597,14 @@ defmodule Plausible.Stats.Base do
)
end
defp total_visitors_subquery(site, query, false) do
defp total_visitors_subquery(site, query) do
dynamic([e], selected_as(subquery(total_visitors(site, query)), :__total_visitors))
end
def add_percentage_metric(q, site, query, metrics) do
if :percentage in metrics do
q
|> select_merge(
^%{__total_visitors: total_visitors_subquery(site, query, query.include_imported)}
)
|> select_merge(^%{__total_visitors: total_visitors_subquery(site, query)})
|> select_merge(%{
percentage:
fragment(
@ -609,32 +618,4 @@ defmodule Plausible.Stats.Base do
q
end
end
# Adds conversion_rate metric to query, calculated as
# X / Y where Y is the same breakdown value without goal or props
# filters.
def maybe_add_conversion_rate(q, site, query, metrics, opts) do
if :conversion_rate in metrics do
include_imported = Keyword.fetch!(opts, :include_imported)
total_query = query |> Query.remove_event_filters([:goal, :props])
# :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALL
subquery(q)
|> select_merge(
^%{total_visitors: total_visitors_subquery(site, total_query, include_imported)}
)
|> select_merge([e], %{
conversion_rate:
fragment(
"if(? > 0, round(? / ? * 100, 1), 0)",
selected_as(:__total_visitors),
e.visitors,
selected_as(:__total_visitors)
)
})
else
q
end
end
end

316
lib/plausible/stats/breakdown.ex
View File

@ -4,7 +4,6 @@ defmodule Plausible.Stats.Breakdown do
use Plausible.Stats.Fragments
import Plausible.Stats.{Base, Imported}
import Ecto.Query
require OpenTelemetry.Tracer, as: Tracer
alias Plausible.Stats.{Query, Util}
@ -53,74 +52,50 @@ defmodule Plausible.Stats.Breakdown do
metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics
event_q =
event_results =
if Enum.any?(event_goals) do
site
|> breakdown_events(event_query, "event:name", metrics_to_select)
|> apply_pagination(pagination)
|> breakdown(event_query, "event:name", metrics_to_select, pagination, skip_tracing: true)
|> transform_keys(%{name: :goal})
|> cast_revenue_metrics_to_money(revenue_goals)
else
nil
[]
end
page_q =
{limit, page} = pagination
offset = (page - 1) * limit
page_results =
if Enum.any?(pageview_goals) do
page_exprs = Enum.map(pageview_goals, & &1.page_path)
page_regexes = Enum.map(page_exprs, &page_regex/1)
select_columns = metrics_to_select |> select_event_metrics |> mark_revenue_as_nil
from(e in base_event_query(site, query),
order_by: [desc: fragment("uniq(?)", e.user_id)],
limit: ^limit,
offset: ^offset,
where:
fragment(
"notEmpty(multiMatchAllIndices(?, ?) as indices)",
e.pathname,
^page_regexes
) and e.name == "pageview",
array_join: index in fragment("indices"),
group_by: index,
select: %{}
group_by: fragment("index"),
select: %{
index: fragment("arrayJoin(indices) as index"),
goal: fragment("concat('Visit ', ?[index])", ^page_exprs)
}
)
|> select_merge(^select_columns)
# :TRICKY: name is added last to make sure both queries add columns in the same order
|> select_merge([_, index], %{
name: fragment("concat('Visit ', ?[?])", ^page_exprs, index)
})
|> apply_pagination(pagination)
|> select_event_metrics(metrics_to_select -- @revenue_metrics)
|> ClickhouseRepo.all()
|> Enum.map(fn row -> Map.delete(row, :index) end)
else
nil
[]
end
full_q =
case {event_q, page_q} do
{nil, nil} ->
nil
{event_q, nil} ->
event_q
{nil, page_q} ->
page_q
{event_q, page_q} ->
from(
e in subquery(union_all(event_q, ^page_q)),
# :TODO: Handle other orderings
order_by: [desc: e.visitors]
)
|> apply_pagination(pagination)
end
if full_q do
full_q
|> maybe_add_conversion_rate(site, query, metrics, include_imported: false)
|> ClickhouseRepo.all()
|> transform_keys(%{name: :goal})
|> cast_revenue_metrics_to_money(revenue_goals)
|> Util.keep_requested_metrics(metrics)
else
[]
end
zip_results(event_results, page_results, :goal, metrics_to_select)
|> maybe_add_cr(site, query, nil, metrics)
|> Util.keep_requested_metrics(metrics)
end
def breakdown(site, query, "event:props:" <> custom_prop = property, metrics, pagination, opts) do
@ -135,11 +110,11 @@ defmodule Plausible.Stats.Breakdown do
if !Keyword.get(opts, :skip_tracing), do: trace(query, property, metrics)
breakdown_events(site, query, "event:props:" <> custom_prop, metrics_to_select)
|> maybe_add_conversion_rate(site, query, metrics, include_imported: false)
|> paginate_and_execute(metrics, pagination)
|> transform_keys(%{name: custom_prop})
breakdown_events(site, query, "event:props:" <> custom_prop, metrics_to_select, pagination)
|> Enum.map(&cast_revenue_metrics_to_money(&1, currency))
|> sort_results(metrics_to_select)
|> maybe_add_cr(site, query, nil, metrics)
|> Util.keep_requested_metrics(metrics)
end
def breakdown(site, query, "event:page" = property, metrics, pagination, opts) do
@ -150,10 +125,10 @@ defmodule Plausible.Stats.Breakdown do
event_result =
site
|> breakdown_events(query, property, event_metrics)
|> maybe_add_group_conversion_rate(&breakdown_events/4, site, query, property, metrics)
|> paginate_and_execute(metrics, pagination)
|> breakdown_events(query, "event:page", event_metrics, pagination)
|> maybe_add_time_on_page(site, query, metrics)
|> maybe_add_cr(site, query, property, metrics, pagination)
|> Util.keep_requested_metrics(metrics)
session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))
@ -174,8 +149,7 @@ defmodule Plausible.Stats.Breakdown do
{limit, _page} = pagination
session_result =
breakdown_sessions(site, new_query, "visit:entry_page", session_metrics)
|> paginate_and_execute(session_metrics, {limit, 1})
breakdown_sessions(site, new_query, "visit:entry_page", session_metrics, {limit, 1})
|> transform_keys(%{entry_page: :page})
metrics = metrics ++ [:page]
@ -192,9 +166,7 @@ defmodule Plausible.Stats.Breakdown do
def breakdown(site, query, "event:name" = property, metrics, pagination, opts) do
if !Keyword.get(opts, :skip_tracing), do: trace(query, property, metrics)
breakdown_events(site, query, property, metrics)
|> paginate_and_execute(metrics, pagination)
breakdown_events(site, query, property, metrics, pagination)
end
def breakdown(site, query, property, metrics, pagination, opts) do
@ -202,9 +174,9 @@ defmodule Plausible.Stats.Breakdown do
metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics
breakdown_sessions(site, query, property, metrics_to_select)
|> maybe_add_group_conversion_rate(&breakdown_sessions/4, site, query, property, metrics)
|> paginate_and_execute(metrics, pagination)
breakdown_sessions(site, query, property, metrics_to_select, pagination)
|> maybe_add_cr(site, query, property, metrics, pagination)
|> Util.keep_requested_metrics(metrics)
end
defp zip_results(event_result, session_result, property, metrics) do
@ -225,31 +197,35 @@ defmodule Plausible.Stats.Breakdown do
|> sort_results(metrics)
end
defp breakdown_sessions(site, query, property, metrics) do
defp breakdown_sessions(_, _, _, [], _), do: []
defp breakdown_sessions(site, query, property, metrics, pagination) do
from(s in query_sessions(site, query),
order_by: [desc: fragment("uniq(?)", s.user_id)],
select: ^select_session_metrics(metrics, query)
select: %{}
)
|> filter_converted_sessions(site, query)
|> do_group_by(property)
|> select_session_metrics(metrics, query)
|> merge_imported(site, query, property, metrics)
|> add_percentage_metric(site, query, metrics)
|> apply_pagination(pagination)
|> ClickhouseRepo.all()
|> transform_keys(%{operating_system: :os})
|> Util.keep_requested_metrics(metrics)
end
defp breakdown_events(site, query, property, metrics) do
defp breakdown_events(_, _, _, [], _), do: []
defp breakdown_events(site, query, property, metrics, pagination) do
from(e in base_event_query(site, query),
order_by: [desc: fragment("uniq(?)", e.user_id)],
select: ^select_event_metrics(metrics)
select: %{}
)
|> do_group_by(property)
|> select_event_metrics(metrics)
|> merge_imported(site, query, property, metrics)
|> add_percentage_metric(site, query, metrics)
end
defp paginate_and_execute(_, [], _), do: []
defp paginate_and_execute(q, metrics, pagination) do
q
|> apply_pagination(pagination)
|> ClickhouseRepo.all()
|> transform_keys(%{operating_system: :os})
@ -447,18 +423,18 @@ defmodule Plausible.Stats.Breakdown do
from(
e in q,
select_merge: %{
name:
^prop =>
selected_as(
fragment(
"if(not empty(?), ?, '(none)')",
get_by_key(e, :meta, ^prop),
get_by_key(e, :meta, ^prop)
),
:name
:breakdown_prop_value
)
},
group_by: selected_as(:name),
order_by: {:asc, selected_as(:name)}
group_by: selected_as(:breakdown_prop_value),
order_by: {:asc, selected_as(:breakdown_prop_value)}
)
end
@ -676,26 +652,130 @@ defmodule Plausible.Stats.Breakdown do
)
end
defp group_by_field_names("event:props:" <> _prop), do: [:name]
defp group_by_field_names("visit:os"), do: [:operating_system]
defp group_by_field_names("visit:os_version"), do: [:os, :os_version]
defp group_by_field_names("visit:browser_version"), do: [:browser, :browser_version]
defp group_by_field_names(property), do: [Plausible.Stats.Filters.without_prefix(property)]
defp on_matches_group_by(fields) do
Enum.reduce(fields, nil, &fields_equal/2)
defp maybe_add_cr(breakdown_results, site, query, property, metrics, pagination \\ nil) do
cond do
:conversion_rate not in metrics -> breakdown_results
Enum.empty?(breakdown_results) -> breakdown_results
is_nil(property) -> add_absolute_cr(breakdown_results, site, query)
true -> add_cr(breakdown_results, site, query, property, metrics, pagination)
end
end
defp outer_order_by(fields) do
Enum.map(fields, fn field_name -> {:asc, dynamic([q], field(q, ^field_name))} end)
# This function injects a conversion_rate metric into every
# breakdown result map. It is calculated as X / Y, where:
#
# * X is the number of conversions for a breakdown
# result (conversion = number of visitors who
# completed the filtered goal with the filtered
# custom properties).
#
# * Y is the number of all visitors for this breakdown
# result without the `event:goal` and `event:props:*`
# filters.
defp add_cr(breakdown_results, site, query, property, metrics, pagination) do
property_atom = Plausible.Stats.Filters.without_prefix(property)
items =
Enum.map(breakdown_results, fn item -> Map.fetch!(item, property_atom) end)
query_without_goal =
query
|> Query.put_filter(property, {:member, items})
|> Query.remove_event_filters([:goal, :props])
# Here, we're always only interested in the first page of results
# - the :member filter makes sure that the results always match with
# the items in the given breakdown_results list
page = 1
# For browser/os versions we need to fetch a lot more entries than the
# pagination limit. This is because many entries can correspond to a
# single version number and we need to make sure that the results
# without goal filter will include all those combinations of browsers/os-s
# and their versions that were present in the `breakdown_results` array.
{pagination_limit, find_match_fn} =
case property_atom do
:browser_version ->
pagination_limit = min(elem(pagination, 0) * 10, 3_000)
find_match_fn = fn total, conversion ->
total[:browser_version] == conversion[:browser_version] &&
total[:browser] == conversion[:browser]
end
{pagination_limit, find_match_fn}
:os_version ->
pagination_limit = min(elem(pagination, 0) * 5, 3_000)
find_match_fn = fn total, conversion ->
total[:os_version] == conversion[:os_version] &&
total[:os] == conversion[:os]
end
{pagination_limit, find_match_fn}
_ ->
{elem(pagination, 0),
fn total, conversion ->
total[property_atom] == conversion[property_atom]
end}
end
pagination = {pagination_limit, page}
res_without_goal = breakdown(site, query_without_goal, property, [:visitors], pagination)
Enum.map(breakdown_results, fn item ->
without_goal = Enum.find(res_without_goal, &find_match_fn.(&1, item))
{conversion_rate, total_visitors} =
if without_goal do
{Util.calculate_cr(without_goal.visitors, item.visitors), without_goal.visitors}
else
Sentry.capture_message(
"Unable to find a conversion_rate divisor from a breakdown response",
extra: %{
domain: site.domain,
query: inspect(query),
property: property,
pagination: inspect(pagination),
item_not_found: inspect(item)
}
)
{"N/A", "N/A"}
end
if :total_visitors in metrics do
item
|> Map.put(:conversion_rate, conversion_rate)
|> Map.put(:total_visitors, total_visitors)
else
Map.put(item, :conversion_rate, conversion_rate)
end
end)
end
defp fields_equal(field_name, nil),
do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name))
# Similar to `add_cr/5`, injects a conversion_rate metric into
# every breakdown result. However, a single divisor is used in
# the CR calculation across all breakdown results. That is the
# number of visitors without `event:goal` and `event:props:*`
# filters.
#
# This is useful when we're only interested in the conversions
# themselves - not how well a certain property such as browser
# or page converted.
defp add_absolute_cr(breakdown_results, site, query) do
total_q = Query.remove_event_filters(query, [:goal, :props])
defp fields_equal(field_name, condition),
do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name) and ^condition)
%{visitors: %{value: total_visitors}} = Plausible.Stats.aggregate(site, total_q, [:visitors])
breakdown_results
|> Enum.map(fn goal ->
Map.put(goal, :conversion_rate, Util.calculate_cr(total_visitors, goal[:visitors]))
end)
end
defp sort_results(results, metrics) do
Enum.sort_by(
@ -710,54 +790,6 @@ defmodule Plausible.Stats.Breakdown do
)
end
# This function injects a conversion_rate metric into
# a breakdown query. It is calculated as X / Y, where:
#
# * X is the number of conversions for a breakdown
# result (conversion = number of visitors who
# completed the filtered goal with the filtered
# custom properties).
#
# * Y is the number of all visitors for this breakdown
# result without the `event:goal` and `event:props:*`
# filters.
defp maybe_add_group_conversion_rate(q, breakdown_fn, site, query, property, metrics) do
if :conversion_rate in metrics do
breakdown_total_visitors_query = query |> Query.remove_event_filters([:goal, :props])
breakdown_total_visitors_q =
breakdown_fn.(site, breakdown_total_visitors_query, property, [:visitors])
from(e in subquery(q),
left_join: c in subquery(breakdown_total_visitors_q),
on: ^on_matches_group_by(group_by_field_names(property)),
select_merge: %{
total_visitors: c.visitors,
conversion_rate:
fragment(
"if(? > 0, round(? / ? * 100, 1), 0)",
c.visitors,
e.visitors,
c.visitors
)
},
order_by: [desc: e.visitors],
order_by: ^outer_order_by(group_by_field_names(property))
)
else
q
end
end
# When querying custom event goals and pageviewgoals together, UNION ALL is used
# so the same fields must be present on both sides of the union. This change to the
# query will ensure that we don't unnecessarily read revenue column for pageview goals
defp mark_revenue_as_nil(select_columns) do
select_columns
|> Map.replace(:total_revenue, nil)
|> Map.replace(:average_revenue, nil)
end
defp sorting_key(metrics) do
if Enum.member?(metrics, :visitors), do: :visitors, else: List.first(metrics)
end
@ -775,8 +807,8 @@ defmodule Plausible.Stats.Breakdown do
offset = (page - 1) * limit
q
|> limit(^limit)
|> offset(^offset)
|> Ecto.Query.limit(^limit)
|> Ecto.Query.offset(^offset)
end
defp trace(query, property, metrics) do

6
lib/plausible/stats/timeseries.ex
View File

@ -53,8 +53,9 @@ defmodule Plausible.Stats.Timeseries do
defp events_timeseries(_, _, []), do: []
defp events_timeseries(site, query, metrics) do
from(e in base_event_query(site, query), select: ^select_event_metrics(metrics))
from(e in base_event_query(site, query), select: %{})
|> select_bucket(site, query)
|> select_event_metrics(metrics)
|> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics)
|> ClickhouseRepo.all()
end
@ -62,9 +63,10 @@ defmodule Plausible.Stats.Timeseries do
defp sessions_timeseries(_, _, []), do: []
defp sessions_timeseries(site, query, metrics) do
from(e in query_sessions(site, query), select: ^select_session_metrics(metrics, query))
from(e in query_sessions(site, query), select: %{})
|> filter_converted_sessions(site, query)
|> select_bucket(site, query)
|> select_session_metrics(metrics, query)
|> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics)
|> ClickhouseRepo.all()
|> Util.keep_requested_metrics(metrics)

11
lib/plausible/stats/util.ex
View File

@ -7,8 +7,7 @@ defmodule Plausible.Stats.Util do
:__internal_visits,
:visitors,
:__total_visitors,
:__breakdown_value,
:total_visitors
:__breakdown_value
]
@doc """
@ -49,4 +48,12 @@ defmodule Plausible.Stats.Util do
metrics
end
end
def calculate_cr(nil, _converted_visitors), do: nil
def calculate_cr(unique_visitors, converted_visitors) do
if unique_visitors > 0,
do: Float.round(converted_visitors / unique_visitors * 100, 1),
else: 0.0
end
end

16
test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
View File

@ -1569,21 +1569,5 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
} =
json_response(conn, 200)["results"]
end
test "conversion_rate for the filtered goal is 0 when no stats exist", %{
conn: conn,
site: site
} do
insert(:goal, %{site: site, event_name: "Signup"})
conn =
get(conn, "/api/v1/stats/aggregate", %{
"site_id" => site.domain,
"metrics" => "conversion_rate",
"filters" => "event:goal==Signup"
})
assert json_response(conn, 200)["results"] == %{"conversion_rate" => %{"value" => 0}}
end
end
end

36
test/plausible_web/controllers/api/stats_controller/conversions_test.exs
View File

@ -580,42 +580,6 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
}
]
end
test "conversion_rate for goals should not be calculated with imported data", %{
conn: conn,
site: site
} do
site =
site
|> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")
|> Plausible.Repo.update!()
populate_stats(site, [
build(:pageview, pathname: "/"),
build(:pageview, pathname: "/another"),
build(:pageview, pathname: "/blog/post-1"),
build(:pageview, pathname: "/blog/post-2"),
build(:imported_pages, page: "/blog/post-1"),
build(:imported_visitors)
])
insert(:goal, %{site: site, page_path: "/blog**"})
conn =
get(
conn,
"/api/stats/#{site.domain}/conversions?period=day"
)
assert json_response(conn, 200) == [
%{
"name" => "Visit /blog**",
"visitors" => 2,
"events" => 2,
"conversion_rate" => 50
}
]
end
end
describe "GET /api/stats/:domain/conversions - with goal and prop=(none) filter" do