plausible/analytics
1
1
Fork 0
mirror of https://github.com/plausible/analytics.git synced 2024-11-22 18:52:38 +03:00
Code Issues Packages Projects Releases Wiki Activity

APIv2: Replace breakdown module with QueryBuilder (#4293)

Browse Source 
* Revert "Revert "APIv2: Replace breakdown module with QueryBuilder (#4283)" (#4292)"

This reverts commit ef5e0e0382.

* Allow querying events and pageviews from sessions table

This is not strictly accurate, especially with shorter time frames, but
is useful for a fallback mechanism. I'll figure out something around
shorter time frames in the future.

See also: https://github.com/plausible/analytics/pull/4292

* Only query events and pageviews in legacy breakdowns
This commit is contained in:
Karl-Aksel Puulmann 2024-07-01 12:50:01 +03:00 committed by GitHub
parent c18ad46212
commit 0594478add
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
25 changed files with 677 additions and 1126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -16,6 +16,7 @@ All notable changes to this project will be documented in this file.
- Increase hourly request limit for API keys in CE from 600 to 1000000 (practically removing the limit) plausible/analytics#4200 - Increase hourly request limit for API keys in CE from 600 to 1000000 (practically removing the limit) plausible/analytics#4200
- Make TCP connections try IPv6 first with IPv4 fallback in CE plausible/analytics#4245 - Make TCP connections try IPv6 first with IPv4 fallback in CE plausible/analytics#4245
- `is` and `is not` filters in dashboard no longer support wildcards. Use contains/does not contain filter instead. - `is` and `is not` filters in dashboard no longer support wildcards. Use contains/does not contain filter instead.
- `bounce_rate` metric now returns 0 instead of null for event:page breakdown when page has never been entry page.
### Fixed ### Fixed

2
lib/plausible/stats/base.ex
View File

@ -250,6 +250,8 @@ defmodule Plausible.Stats.Base do
end end
defp select_session_metric(:percentage, _query), do: %{} defp select_session_metric(:percentage, _query), do: %{}
defp select_session_metric(:conversion_rate, _query), do: %{}
defp select_session_metric(:group_conversion_rate, _query), do: %{}
def filter_converted_sessions(db_query, site, query) do def filter_converted_sessions(db_query, site, query) do
if Query.has_event_filters?(query) do if Query.has_event_filters?(query) do

786
lib/plausible/stats/breakdown.ex
View File

@ -3,344 +3,63 @@ defmodule Plausible.Stats.Breakdown do
use Plausible use Plausible
use Plausible.Stats.Fragments use Plausible.Stats.Fragments
import Plausible.Stats.{Base, Imported} import Plausible.Stats.Base
import Ecto.Query import Ecto.Query
alias Plausible.Stats.{Query, Util, TableDecider} alias Plausible.Stats.{Query, QueryOptimizer, QueryResult, SQL}
alias Plausible.Stats.Filters.QueryParser
@no_ref "Direct / None" def breakdown(site, %Query{dimensions: [dimension]} = query, metrics, pagination, _opts \\ []) do
@not_set "(not set)" transformed_metrics = transform_metrics(metrics, dimension)
@session_metrics [:bounce_rate, :visit_duration] query_with_metrics =
Query.set(
@revenue_metrics on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: []) query,
metrics: transformed_metrics,
@event_metrics [:visits, :visitors, :pageviews, :events, :percentage] ++ @revenue_metrics order_by: infer_order_by(transformed_metrics, dimension),
dimensions: transform_dimensions(dimension),
# These metrics can be asked from the `breakdown/5` function, filters: query.filters ++ dimension_filters(dimension),
# but they are different from regular metrics such as `visitors`, preloaded_goals: QueryParser.preload_goals_if_needed(site, query.filters, [dimension]),
# or `bounce_rate` - we cannot currently "select them" directly in v2: true,
# the db queries. Instead, we need to artificially append them to # Allow pageview and event metrics to be queried off of sessions table
# the breakdown results later on. legacy_breakdown: true
@computed_metrics [:conversion_rate, :total_visitors]
def breakdown(site, query, metrics, pagination, opts \\ [])
def breakdown(
site,
%Query{dimensions: ["event:goal"]} = query,
metrics,
pagination,
opts
) do
site = Plausible.Repo.preload(site, :goals)
{event_goals, pageview_goals} = Enum.split_with(site.goals, & &1.event_name)
events = Enum.map(event_goals, & &1.event_name)
event_query =
query
|> Query.put_filter([:is, "event:name", events])
|> Query.set_dimensions(["event:name"])
if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)
no_revenue = {nil, metrics -- @revenue_metrics}
{revenue_goals, metrics} =
on_ee do
if Plausible.Billing.Feature.RevenueGoals.enabled?(site) do
revenue_goals = Enum.filter(event_goals, &Plausible.Goal.Revenue.revenue?/1)
metrics = if Enum.empty?(revenue_goals), do: metrics -- @revenue_metrics, else: metrics
{revenue_goals, metrics}
else
no_revenue
end
else
no_revenue
end
metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics
event_q =
if Enum.any?(event_goals) do
site
|> breakdown_events(event_query, metrics_to_select)
|> apply_pagination(pagination)
else
nil
end
page_q =
if Enum.any?(pageview_goals) do
page_query = Query.set_dimensions(query, ["event:page"])
page_exprs = Enum.map(pageview_goals, & &1.page_path)
page_regexes = Enum.map(page_exprs, &page_regex/1)
select_columns = metrics_to_select |> select_event_metrics |> mark_revenue_as_nil
from(e in base_event_query(site, page_query),
order_by: [desc: fragment("uniq(?)", e.user_id)],
where:
fragment(
"notEmpty(multiMatchAllIndices(?, ?) as indices)",
e.pathname,
^page_regexes
) and e.name == "pageview",
array_join: index in fragment("indices"),
group_by: index,
select: %{
name: fragment("concat('Visit ', ?[?])", ^page_exprs, index)
}
)
|> select_merge(^select_columns)
|> merge_imported_pageview_goals(site, page_query, page_exprs, metrics_to_select)
|> apply_pagination(pagination)
else
nil
end
full_q =
case {event_q, page_q} do
{nil, nil} ->
nil
{event_q, nil} ->
event_q
{nil, page_q} ->
page_q
{event_q, page_q} ->
from(
e in subquery(union_all(event_q, ^page_q)),
order_by: [desc: e.visitors]
)
|> apply_pagination(pagination)
end
if full_q do
full_q
|> maybe_add_conversion_rate(site, query, metrics)
|> ClickhouseRepo.all()
|> transform_keys(%{name: :goal})
|> cast_revenue_metrics_to_money(revenue_goals)
|> Util.keep_requested_metrics(metrics)
else
[]
end
end
def breakdown(
site,
%Query{dimensions: ["event:props:" <> custom_prop]} = query,
metrics,
pagination,
opts
) do
{currency, metrics} =
on_ee do
Plausible.Stats.Goal.Revenue.get_revenue_tracking_currency(site, query, metrics)
else
{nil, metrics}
end
metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics
if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)
breakdown_events(site, query, metrics_to_select)
|> maybe_add_conversion_rate(site, query, metrics)
|> paginate_and_execute(metrics, pagination)
|> transform_keys(%{breakdown_prop_value: custom_prop})
|> Enum.map(&cast_revenue_metrics_to_money(&1, currency))
end
def breakdown(site, %Query{dimensions: ["event:page"]} = query, metrics, pagination, opts) do
event_metrics =
metrics
|> Util.maybe_add_visitors_metric()
|> Enum.filter(&(&1 in @event_metrics))
if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)
event_result =
site
|> breakdown_events(query, event_metrics)
|> maybe_add_group_conversion_rate(&breakdown_events/3, site, query, metrics)
|> paginate_and_execute(metrics, pagination)
|> maybe_add_time_on_page(site, query, metrics)
session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))
entry_page_query =
case event_result do
[] ->
query
pages ->
query
|> Query.remove_filters(["event:page"])
|> Query.put_filter([:is, "visit:entry_page", Enum.map(pages, & &1[:page])])
|> Query.set_dimensions(["visit:entry_page"])
end
if Enum.any?(event_metrics) && Enum.empty?(event_result) do
[]
else
{limit, _page} = pagination
session_result =
breakdown_sessions(site, entry_page_query, session_metrics)
|> paginate_and_execute(session_metrics, {limit, 1})
|> transform_keys(%{entry_page: :page})
metrics = metrics ++ [:page]
zip_results(
event_result,
session_result,
:page,
metrics
) )
|> Enum.map(&Map.take(&1, metrics)) |> QueryOptimizer.optimize()
end
end
def breakdown(site, %Query{dimensions: ["event:name"]} = query, metrics, pagination, opts) do q = SQL.QueryBuilder.build(query_with_metrics, site)
if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)
breakdown_events(site, query, metrics)
|> paginate_and_execute(metrics, pagination)
end
def breakdown(site, query, metrics, pagination, opts) do
query = maybe_update_breakdown_filters(query)
if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)
metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics
case breakdown_table(query, metrics) do
:session ->
breakdown_sessions(site, query, metrics_to_select)
|> maybe_add_group_conversion_rate(&breakdown_sessions/3, site, query, metrics)
|> paginate_and_execute(metrics, pagination)
:event ->
breakdown_events(site, query, metrics_to_select)
|> maybe_add_group_conversion_rate(&breakdown_events/3, site, query, metrics)
|> paginate_and_execute(metrics, pagination)
end
end
defp maybe_update_breakdown_filters(%Query{dimensions: [visit_entry_prop]} = query)
when visit_entry_prop in [
"visit:source",
"visit:entry_page",
"visit:utm_medium",
"visit:utm_source",
"visit:utm_campaign",
"visit:utm_content",
"visit:utm_term",
"visit:entry_page",
"visit:referrer"
] do
update_hostname_filter_prop(query, "visit:entry_page_hostname")
end
defp maybe_update_breakdown_filters(%Query{dimensions: ["visit:exit_page"]} = query) do
update_hostname_filter_prop(query, "visit:exit_page_hostname")
end
defp maybe_update_breakdown_filters(query) do
query
end
defp update_hostname_filter_prop(query, visit_prop) do
case Query.get_filter(query, "event:hostname") do
nil ->
query
[op, "event:hostname", value] ->
query
|> Query.put_filter([op, visit_prop, value])
end
end
# Backwards compatibility
defp breakdown_table(%Query{experimental_reduced_joins?: false}, _), do: :session
defp breakdown_table(%Query{dimensions: ["visit:entry_page"]}, _metrics), do: :session
defp breakdown_table(%Query{dimensions: ["visit:entry_page_hostname"]}, _metrics), do: :session
defp breakdown_table(%Query{dimensions: ["visit:exit_page"]}, _metrics), do: :session
defp breakdown_table(%Query{dimensions: ["visit:exit_page_hostname"]}, _metrics), do: :session
defp breakdown_table(%Query{dimensions: [_dimension]} = query, metrics) do
{_, session_metrics, _} = TableDecider.partition_metrics(metrics, query)
if not Enum.empty?(session_metrics) do
:session
else
:event
end
end
defp zip_results(event_result, session_result, property, metrics) do
null_row = Enum.map(metrics, fn metric -> {metric, nil} end) |> Enum.into(%{})
prop_values =
Enum.map(event_result ++ session_result, fn row -> row[property] end)
|> Enum.uniq()
Enum.map(prop_values, fn value ->
event_row = Enum.find(event_result, fn row -> row[property] == value end) || %{}
session_row = Enum.find(session_result, fn row -> row[property] == value end) || %{}
null_row
|> Map.merge(event_row)
|> Map.merge(session_row)
end)
|> sort_results(metrics)
end
defp breakdown_sessions(site, %Query{dimensions: [dimension]} = query, metrics) do
from(s in query_sessions(site, query),
order_by: [desc: fragment("uniq(?)", s.user_id)],
select: ^select_session_metrics(metrics, query)
)
|> filter_converted_sessions(site, query)
|> do_group_by(dimension)
|> merge_imported(site, query, metrics)
|> add_percentage_metric(site, query, metrics)
end
defp breakdown_events(site, %Query{dimensions: [dimension]} = query, metrics) do
from(e in base_event_query(site, query),
order_by: [desc: fragment("uniq(?)", e.user_id)],
select: %{}
)
|> do_group_by(dimension)
|> select_merge(^select_event_metrics(metrics))
|> merge_imported(site, query, metrics)
|> add_percentage_metric(site, query, metrics)
end
defp paginate_and_execute(_, [], _), do: []
defp paginate_and_execute(q, metrics, pagination) do
q q
|> apply_pagination(pagination) |> apply_pagination(pagination)
|> ClickhouseRepo.all() |> ClickhouseRepo.all()
|> Util.keep_requested_metrics(metrics) |> QueryResult.from(query_with_metrics)
|> build_breakdown_result(query_with_metrics, metrics)
|> maybe_add_time_on_page(site, query_with_metrics, metrics)
|> update_currency_metrics(site, query_with_metrics)
end end
defp build_breakdown_result(query_result, query, metrics) do
query_result.results
|> Enum.map(fn %{dimensions: dimensions, metrics: entry_metrics} ->
dimension_map =
query.dimensions |> Enum.map(&result_key/1) |> Enum.zip(dimensions) |> Enum.into(%{})
metrics_map = Enum.zip(metrics, entry_metrics) |> Enum.into(%{})
Map.merge(dimension_map, metrics_map)
end)
end
defp result_key("event:props:" <> custom_property), do: custom_property
defp result_key("event:" <> key), do: key |> String.to_existing_atom()
defp result_key("visit:" <> key), do: key |> String.to_existing_atom()
defp result_key(dimension), do: dimension
defp maybe_add_time_on_page(event_results, site, query, metrics) do defp maybe_add_time_on_page(event_results, site, query, metrics) do
if :time_on_page in metrics do if query.dimensions == ["event:page"] and :time_on_page in metrics do
pages = Enum.map(event_results, & &1[:page]) pages = Enum.map(event_results, & &1[:page])
time_on_page_result = breakdown_time_on_page(site, query, pages) time_on_page_result = breakdown_time_on_page(site, query, pages)
Enum.map(event_results, fn row -> event_results
|> Enum.map(fn row ->
Map.put(row, :time_on_page, time_on_page_result[row[:page]]) Map.put(row, :time_on_page, time_on_page_result[row[:page]])
end) end)
else else
@ -431,346 +150,60 @@ defmodule Plausible.Stats.Breakdown do
|> Map.new() |> Map.new()
end end
defp do_group_by( defp transform_metrics(metrics, dimension) do
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q, metrics =
"event:props:" <> prop if is_nil(metric_to_order_by(metrics)) do
) do metrics ++ [:visitors]
from( else
e in q, metrics
select_merge: %{ end
breakdown_prop_value:
selected_as(
fragment(
"if(not empty(?), ?, '(none)')",
get_by_key(e, :meta, ^prop),
get_by_key(e, :meta, ^prop)
),
:breakdown_prop_value
)
},
group_by: selected_as(:breakdown_prop_value),
order_by: {:asc, selected_as(:breakdown_prop_value)}
)
end
defp do_group_by( Enum.map(metrics, fn metric ->
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q, case {metric, dimension} do
"event:name" {:conversion_rate, "event:props:" <> _} -> :conversion_rate
) do {:conversion_rate, "event:goal"} -> :conversion_rate
from( {:conversion_rate, _} -> :group_conversion_rate
e in q, _ -> metric
group_by: e.name, end
select_merge: %{name: e.name},
order_by: {:asc, e.name}
)
end
defp do_group_by(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,
"event:page"
) do
from(
e in q,
group_by: e.pathname,
select_merge: %{page: e.pathname},
order_by: {:asc, e.pathname}
)
end
defp do_group_by(q, "visit:source") do
from(
s in q,
group_by: s.source,
select_merge: %{
source: fragment("if(empty(?), ?, ?)", s.source, @no_ref, s.source)
},
order_by: {:asc, s.source}
)
end
defp do_group_by(q, "visit:country") do
from(
s in q,
where: s.country != "\0\0" and s.country != "ZZ",
group_by: s.country,
select_merge: %{country: s.country},
order_by: {:asc, s.country}
)
end
defp do_group_by(q, "visit:region") do
from(
s in q,
where: s.region != "",
group_by: s.region,
select_merge: %{region: s.region},
order_by: {:asc, s.region}
)
end
defp do_group_by(q, "visit:city") do
from(
s in q,
where: s.city != 0,
group_by: s.city,
select_merge: %{city: s.city},
order_by: {:asc, s.city}
)
end
defp do_group_by(q, "visit:entry_page") do
from(
s in q,
# Sessions without pageviews don't get entry_page assigned, hence they should get ignored
where: s.entry_page != "",
group_by: s.entry_page,
select_merge: %{entry_page: s.entry_page},
order_by: {:asc, s.entry_page}
)
end
defp do_group_by(q, "visit:exit_page") do
from(
s in q,
# Sessions without pageviews don't get entry_page assigned, hence they should get ignored
where: s.entry_page != "",
group_by: s.exit_page,
select_merge: %{exit_page: s.exit_page},
order_by: {:asc, s.exit_page}
)
end
defp do_group_by(q, "visit:referrer") do
from(
s in q,
group_by: s.referrer,
select_merge: %{
referrer: fragment("if(empty(?), ?, ?)", s.referrer, @no_ref, s.referrer)
},
order_by: {:asc, s.referrer}
)
end
defp do_group_by(q, "visit:utm_medium") do
from(
s in q,
where: fragment("not empty(?)", s.utm_medium),
group_by: s.utm_medium,
select_merge: %{
utm_medium: s.utm_medium
}
)
end
defp do_group_by(q, "visit:utm_source") do
from(
s in q,
where: fragment("not empty(?)", s.utm_source),
group_by: s.utm_source,
select_merge: %{
utm_source: s.utm_source
}
)
end
defp do_group_by(q, "visit:utm_campaign") do
from(
s in q,
where: fragment("not empty(?)", s.utm_campaign),
group_by: s.utm_campaign,
select_merge: %{
utm_campaign: s.utm_campaign
}
)
end
defp do_group_by(q, "visit:utm_content") do
from(
s in q,
where: fragment("not empty(?)", s.utm_content),
group_by: s.utm_content,
select_merge: %{
utm_content: s.utm_content
}
)
end
defp do_group_by(q, "visit:utm_term") do
from(
s in q,
where: fragment("not empty(?)", s.utm_term),
group_by: s.utm_term,
select_merge: %{
utm_term: s.utm_term
}
)
end
defp do_group_by(q, "visit:device") do
from(
s in q,
group_by: s.device,
select_merge: %{
device: fragment("if(empty(?), ?, ?)", s.device, @not_set, s.device)
},
order_by: {:asc, s.device}
)
end
defp do_group_by(q, "visit:os") do
from(
s in q,
group_by: s.os,
select_merge: %{
os: fragment("if(empty(?), ?, ?)", s.os, @not_set, s.os)
},
order_by: {:asc, s.os}
)
end
defp do_group_by(q, "visit:os_version") do
from(
s in q,
group_by: [s.os, s.os_version],
select_merge: %{
os: fragment("if(empty(?), ?, ?)", s.os, @not_set, s.os),
os_version:
fragment(
"if(empty(?), ?, ?)",
s.os_version,
@not_set,
s.os_version
)
},
order_by: {:asc, s.os_version}
)
end
defp do_group_by(q, "visit:browser") do
from(
s in q,
group_by: s.browser,
select_merge: %{
browser: fragment("if(empty(?), ?, ?)", s.browser, @not_set, s.browser)
},
order_by: {:asc, s.browser}
)
end
defp do_group_by(q, "visit:browser_version") do
from(
s in q,
group_by: [s.browser, s.browser_version],
select_merge: %{
browser: fragment("if(empty(?), ?, ?)", s.browser, @not_set, s.browser),
browser_version:
fragment("if(empty(?), ?, ?)", s.browser_version, @not_set, s.browser_version)
},
order_by: {:asc, s.browser_version}
)
end
defp group_by_field_names("event:props:" <> _prop), do: [:name]
defp group_by_field_names("visit:os_version"), do: [:os, :os_version]
defp group_by_field_names("visit:browser_version"), do: [:browser, :browser_version]
defp group_by_field_names(property), do: [Plausible.Stats.Filters.without_prefix(property)]
defp on_matches_group_by(fields) do
Enum.reduce(fields, nil, &fields_equal/2)
end
defp outer_order_by(fields) do
Enum.map(fields, fn field_name -> {:asc, dynamic([q], field(q, ^field_name))} end)
end
defp fields_equal(field_name, nil),
do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name))
defp fields_equal(field_name, condition),
do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name) and ^condition)
defp sort_results(results, metrics) do
Enum.sort_by(
results,
fn entry ->
case entry[sorting_key(metrics)] do
nil -> 0
n -> n
end
end,
:desc
)
end
# This function injects a conversion_rate metric into
# a breakdown query. It is calculated as X / Y, where:
#
# * X is the number of conversions for a breakdown
# result (conversion = number of visitors who
# completed the filtered goal with the filtered
# custom properties).
#
# * Y is the number of all visitors for this breakdown
# result without the `event:goal` and `event:props:*`
# filters.
defp maybe_add_group_conversion_rate(
q,
breakdown_fn,
site,
%Query{dimensions: [dimension]} = query,
metrics
) do
if :conversion_rate in metrics do
breakdown_total_visitors_query =
query |> Query.remove_filters(["event:goal", "event:props"])
breakdown_total_visitors_q =
breakdown_fn.(site, breakdown_total_visitors_query, [:visitors])
from(e in subquery(q),
left_join: c in subquery(breakdown_total_visitors_q),
on: ^on_matches_group_by(group_by_field_names(dimension)),
select_merge: %{
total_visitors: c.visitors,
conversion_rate:
fragment(
"if(? > 0, round(? / ? * 100, 1), 0)",
c.visitors,
e.visitors,
c.visitors
)
},
order_by: [desc: e.visitors],
order_by: ^outer_order_by(group_by_field_names(dimension))
)
else
q
end
end
# When querying custom event goals and pageviewgoals together, UNION ALL is used
# so the same fields must be present on both sides of the union. This change to the
# query will ensure that we don't unnecessarily read revenue column for pageview goals
defp mark_revenue_as_nil(select_columns) do
select_columns
|> Map.replace(:total_revenue, nil)
|> Map.replace(:average_revenue, nil)
end
defp sorting_key(metrics) do
if Enum.member?(metrics, :visitors), do: :visitors, else: List.first(metrics)
end
defp transform_keys(results, keys_to_replace) do
Enum.map(results, fn map ->
Enum.map(map, fn {key, val} ->
{Map.get(keys_to_replace, key, key), val}
end)
|> Enum.into(%{})
end) end)
end end
defp infer_order_by(metrics, "event:goal"), do: [{metric_to_order_by(metrics), :desc}]
defp infer_order_by(metrics, dimension),
do: [{metric_to_order_by(metrics), :desc}, {dimension, :asc}]
defp metric_to_order_by(metrics) do
Enum.find(metrics, &(&1 != :time_on_page))
end
def transform_dimensions("visit:browser_version"),
do: ["visit:browser", "visit:browser_version"]
def transform_dimensions("visit:os_version"), do: ["visit:os", "visit:os_version"]
def transform_dimensions(dimension), do: [dimension]
@filter_dimensions_not %{
"visit:city" => [0],
"visit:country" => ["\0\0", "ZZ"],
"visit:region" => [""],
"visit:utm_medium" => [""],
"visit:utm_source" => [""],
"visit:utm_campaign" => [""],
"visit:utm_content" => [""],
"visit:utm_term" => [""],
"visit:entry_page" => [""],
"visit:exit_page" => [""]
}
@extra_filter_dimensions Map.keys(@filter_dimensions_not)
defp dimension_filters(dimension) when dimension in @extra_filter_dimensions do
[[:is_not, dimension, Map.get(@filter_dimensions_not, dimension)]]
end
defp dimension_filters(_), do: []
defp apply_pagination(q, {limit, page}) do defp apply_pagination(q, {limit, page}) do
offset = (page - 1) * limit offset = (page - 1) * limit
@ -780,10 +213,39 @@ defmodule Plausible.Stats.Breakdown do
end end
on_ee do on_ee do
defp cast_revenue_metrics_to_money(results, revenue_goals) do defp update_currency_metrics(results, site, %Query{dimensions: ["event:goal"]}) do
Plausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(results, revenue_goals) site = Plausible.Repo.preload(site, :goals)
{event_goals, _pageview_goals} = Enum.split_with(site.goals, & &1.event_name)
revenue_goals = Enum.filter(event_goals, &Plausible.Goal.Revenue.revenue?/1)
if length(revenue_goals) > 0 and Plausible.Billing.Feature.RevenueGoals.enabled?(site) do
Plausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(results, revenue_goals)
else
remove_revenue_metrics(results)
end
end
defp update_currency_metrics(results, site, query) do
{currency, _metrics} =
Plausible.Stats.Goal.Revenue.get_revenue_tracking_currency(site, query, query.metrics)
if currency do
results
|> Enum.map(&Plausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(&1, currency))
else
remove_revenue_metrics(results)
end
end end
else else
defp cast_revenue_metrics_to_money(results, _revenue_goals), do: results defp update_currency_metrics(results, _site, _query), do: remove_revenue_metrics(results)
end
defp remove_revenue_metrics(results) do
Enum.map(results, fn map ->
map
|> Map.delete(:total_revenue)
|> Map.delete(:average_revenue)
end)
end end
end end

20
lib/plausible/stats/filters/query_parser.ex
View File

@ -4,6 +4,7 @@ defmodule Plausible.Stats.Filters.QueryParser do
alias Plausible.Stats.TableDecider alias Plausible.Stats.TableDecider
alias Plausible.Stats.Filters alias Plausible.Stats.Filters
alias Plausible.Stats.Query alias Plausible.Stats.Query
alias Plausible.Stats.Metrics
def parse(site, params, now \\ nil) when is_map(params) do def parse(site, params, now \\ nil) when is_map(params) do
with {:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])), with {:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])),
@ -44,17 +45,12 @@ defmodule Plausible.Stats.Filters.QueryParser do
defp parse_metrics(_invalid_metrics), do: {:error, "Invalid metrics passed"} defp parse_metrics(_invalid_metrics), do: {:error, "Invalid metrics passed"}
defp parse_metric("time_on_page"), do: {:ok, :time_on_page} defp parse_metric(metric_str) do
defp parse_metric("conversion_rate"), do: {:ok, :conversion_rate} case Metrics.from_string(metric_str) do
defp parse_metric("group_conversion_rate"), do: {:ok, :group_conversion_rate} {:ok, metric} -> {:ok, metric}
defp parse_metric("visitors"), do: {:ok, :visitors} _ -> {:error, "Unknown metric '#{inspect(metric_str)}'"}
defp parse_metric("pageviews"), do: {:ok, :pageviews} end
defp parse_metric("events"), do: {:ok, :events} end
defp parse_metric("visits"), do: {:ok, :visits}
defp parse_metric("bounce_rate"), do: {:ok, :bounce_rate}
defp parse_metric("visit_duration"), do: {:ok, :visit_duration}
defp parse_metric("views_per_visit"), do: {:ok, :views_per_visit}
defp parse_metric(unknown_metric), do: {:error, "Unknown metric '#{inspect(unknown_metric)}'"}
def parse_filters(filters) when is_list(filters) do def parse_filters(filters) when is_list(filters) do
parse_list(filters, &parse_filter/1) parse_list(filters, &parse_filter/1)
@ -284,7 +280,7 @@ defmodule Plausible.Stats.Filters.QueryParser do
end end
end end
defp preload_goals_if_needed(site, filters, dimensions) do def preload_goals_if_needed(site, filters, dimensions) do
goal_filters? = goal_filters? =
Enum.any?(filters, fn [_, filter_key | _rest] -> filter_key == "event:goal" end) Enum.any?(filters, fn [_, filter_key | _rest] -> filter_key == "event:goal" end)

11
lib/plausible/stats/filters/where_builder.ex
View File

@ -10,6 +10,8 @@ defmodule Plausible.Stats.Filters.WhereBuilder do
use Plausible.Stats.Fragments use Plausible.Stats.Fragments
require Logger
@sessions_only_visit_fields [ @sessions_only_visit_fields [
:entry_page, :entry_page,
:exit_page, :exit_page,
@ -150,6 +152,15 @@ defmodule Plausible.Stats.Filters.WhereBuilder do
true true
end end
defp add_filter(table, _query, filter) do
Logger.info("Unable to process garbage filter. No results are returned",
table: table,
filter: filter
)
false
end
defp filter_custom_prop(prop_name, column_name, [:is, _, values]) do defp filter_custom_prop(prop_name, column_name, [:is, _, values]) do
none_value_included = Enum.member?(values, "(none)") none_value_included = Enum.member?(values, "(none)")

77
lib/plausible/stats/imported/base.ex
View File

@ -54,10 +54,12 @@ defmodule Plausible.Stats.Imported.Base do
def property_to_table_mappings(), do: @property_to_table_mappings def property_to_table_mappings(), do: @property_to_table_mappings
def query_imported(site, query) do def query_imported(site, query) do
query [table] =
|> transform_filters() query
|> decide_table() |> transform_filters()
|> query_imported(site, query) |> decide_tables()
query_imported(table, site, query)
end end
def query_imported(table, site, query) do def query_imported(table, site, query) do
@ -75,13 +77,13 @@ defmodule Plausible.Stats.Imported.Base do
|> apply_filter(query) |> apply_filter(query)
end end
def decide_table(query) do def decide_tables(query) do
query = transform_filters(query) query = transform_filters(query)
if custom_prop_query?(query) do if custom_prop_query?(query) do
do_decide_custom_prop_table(query) do_decide_custom_prop_table(query)
else else
do_decide_table(query) do_decide_tables(query)
end end
end end
@ -92,19 +94,17 @@ defmodule Plausible.Stats.Imported.Base do
[:is, "event:name", ["pageview"]] -> true [:is, "event:name", ["pageview"]] -> true
_ -> false _ -> false
end) end)
|> Enum.flat_map(fn filter -> |> Enum.flat_map(fn
case filter do [op, "event:goal", clauses] ->
[op, "event:goal", events] -> clauses
events |> Enum.group_by(&elem(&1, 0), &elem(&1, 1))
|> Enum.group_by(&elem(&1, 0), &elem(&1, 1)) |> Enum.map(fn
|> Enum.map(fn {:event, names} -> [op, "event:name", names]
{:event, names} -> [op, "event:name", names] {:page, pages} -> [op, "event:page", pages]
{:page, pages} -> [op, "event:page", pages] end)
end)
filter -> filter ->
[filter] [filter]
end
end) end)
struct!(query, filters: new_filters) struct!(query, filters: new_filters)
@ -136,10 +136,10 @@ defmodule Plausible.Stats.Imported.Base do
do_decide_custom_prop_table(query, custom_prop_filter) do_decide_custom_prop_table(query, custom_prop_filter)
_ -> _ ->
nil []
end end
else else
nil []
end end
end end
@ -158,23 +158,27 @@ defmodule Plausible.Stats.Imported.Base do
end) end)
if has_required_name_filter? and not has_unsupported_filters? do if has_required_name_filter? and not has_unsupported_filters? do
"imported_custom_events" ["imported_custom_events"]
else else
nil []
end end
end end
defp do_decide_table(%Query{filters: [], dimensions: []}), do: "imported_visitors" defp do_decide_tables(%Query{filters: [], dimensions: []}), do: ["imported_visitors"]
defp do_decide_table(%Query{filters: [], dimensions: ["event:goal"]}) do defp do_decide_tables(%Query{filters: [], dimensions: ["event:goal"]}) do
"imported_custom_events" ["imported_pages", "imported_custom_events"]
end end
defp do_decide_table(%Query{filters: [], dimensions: [dimension]}) do defp do_decide_tables(%Query{filters: [], dimensions: [dimension]}) do
@property_to_table_mappings[dimension] if Map.has_key?(@property_to_table_mappings, dimension) do
[@property_to_table_mappings[dimension]]
else
[]
end
end end
defp do_decide_table(%Query{filters: filters, dimensions: ["event:goal"]}) do defp do_decide_tables(%Query{filters: filters, dimensions: ["event:goal"]}) do
filter_props = Enum.map(filters, &Enum.at(&1, 1)) filter_props = Enum.map(filters, &Enum.at(&1, 1))
any_event_name_filters? = "event:name" in filter_props any_event_name_filters? = "event:name" in filter_props
@ -182,27 +186,28 @@ defmodule Plausible.Stats.Imported.Base do
any_other_filters? = Enum.any?(filter_props, &(&1 not in ["event:page", "event:name"])) any_other_filters? = Enum.any?(filter_props, &(&1 not in ["event:page", "event:name"]))
cond do cond do
any_other_filters? -> nil any_other_filters? -> []
any_event_name_filters? and not any_page_filters? -> "imported_custom_events" any_event_name_filters? and not any_page_filters? -> ["imported_custom_events"]
any_page_filters? and not any_event_name_filters? -> "imported_pages" any_page_filters? and not any_event_name_filters? -> ["imported_pages"]
true -> nil true -> []
end end
end end
defp do_decide_table(%Query{filters: filters, dimensions: dimensions}) do defp do_decide_tables(%Query{filters: filters, dimensions: dimensions}) do
table_candidates = table_candidates =
filters filters
|> Enum.map(fn [_, filter_key | _] -> filter_key end) |> Enum.map(fn [_, filter_key | _] -> filter_key end)
|> Enum.concat(dimensions) |> Enum.concat(dimensions)
|> Enum.map(fn |> Enum.map(fn
"visit:screen" -> "visit:device" "visit:screen" -> "visit:device"
prop -> prop dimension -> dimension
end) end)
|> Enum.map(&@property_to_table_mappings[&1]) |> Enum.map(&@property_to_table_mappings[&1])
case Enum.uniq(table_candidates) do case Enum.uniq(table_candidates) do
[candidate] -> candidate [nil] -> []
_ -> nil [candidate] -> [candidate]
_ -> []
end end
end end

366
lib/plausible/stats/imported/imported.ex
View File

@ -4,10 +4,11 @@ defmodule Plausible.Stats.Imported do
import Ecto.Query import Ecto.Query
import Plausible.Stats.Fragments import Plausible.Stats.Fragments
import Plausible.Stats.Util, only: [shortname: 2]
alias Plausible.Stats.Base
alias Plausible.Stats.Imported alias Plausible.Stats.Imported
alias Plausible.Stats.Query alias Plausible.Stats.Query
alias Plausible.Stats.SQL.QueryBuilder
@no_ref "Direct / None" @no_ref "Direct / None"
@not_set "(not set)" @not_set "(not set)"
@ -15,7 +16,7 @@ defmodule Plausible.Stats.Imported do
@property_to_table_mappings Imported.Base.property_to_table_mappings() @property_to_table_mappings Imported.Base.property_to_table_mappings()
@imported_properties Map.keys(@property_to_table_mappings) ++ @imported_dimensions Map.keys(@property_to_table_mappings) ++
Plausible.Imported.imported_custom_props() Plausible.Imported.imported_custom_props()
@goals_with_url Plausible.Imported.goals_with_url() @goals_with_url Plausible.Imported.goals_with_url()
@ -37,7 +38,7 @@ defmodule Plausible.Stats.Imported do
(see `@goals_with_url` and `@goals_with_path`). (see `@goals_with_url` and `@goals_with_path`).
""" """
def schema_supports_query?(query) do def schema_supports_query?(query) do
not is_nil(Imported.Base.decide_table(query)) length(Imported.Base.decide_tables(query)) > 0
end end
def merge_imported_country_suggestions(native_q, _site, %Plausible.Stats.Query{ def merge_imported_country_suggestions(native_q, _site, %Plausible.Stats.Query{
@ -267,78 +268,6 @@ defmodule Plausible.Stats.Imported do
def merge_imported(q, _, %Query{include_imported: false}, _), do: q def merge_imported(q, _, %Query{include_imported: false}, _), do: q
# Note: Only called for APIv2, old APIs use merge_imported_pageview_goals
def merge_imported(q, site, %Query{dimensions: ["event:goal"]} = query, metrics)
when query.v2 do
{events, page_regexes} = Filters.Utils.split_goals_query_expressions(query.preloaded_goals)
events_q =
"imported_custom_events"
|> Imported.Base.query_imported(site, query)
|> where([i], i.visitors > 0)
|> select_merge([i], %{
dim0: selected_as(fragment("-indexOf(?, ?)", ^events, i.name), :dim0)
})
|> select_imported_metrics(metrics)
|> group_by([], selected_as(:dim0))
|> where([], selected_as(:dim0) != 0)
pages_q =
"imported_pages"
|> Imported.Base.query_imported(site, query)
|> where([i], i.visitors > 0)
|> where(
[i],
fragment("notEmpty(multiMatchAllIndices(?, ?) as indices)", i.page, ^page_regexes)
)
|> join(:array, index in fragment("indices"))
|> group_by([_i, index], index)
|> select_merge([_i, index], %{
dim0: type(fragment("?", index), :integer)
})
|> select_imported_metrics(metrics)
q
|> naive_dimension_join(events_q, metrics)
|> naive_dimension_join(pages_q, metrics)
end
def merge_imported(q, site, %Query{dimensions: [dimension]} = query, metrics)
when dimension in @imported_properties do
dim = Plausible.Stats.Filters.without_prefix(dimension)
imported_q =
site
|> Imported.Base.query_imported(query)
|> where([i], i.visitors > 0)
|> group_imported_by(dim, query)
|> select_imported_metrics(metrics)
join_on =
case dim do
_ when dim in [:url, :path] and not query.v2 ->
dynamic([s, i], s.breakdown_prop_value == i.breakdown_prop_value)
:os_version when not query.v2 ->
dynamic([s, i], s.os == i.os and s.os_version == i.os_version)
:browser_version when not query.v2 ->
dynamic([s, i], s.browser == i.browser and s.browser_version == i.browser_version)
dim ->
dynamic([s, i], field(s, ^shortname(query, dim)) == field(i, ^shortname(query, dim)))
end
from(s in Ecto.Query.subquery(q),
full_join: i in subquery(imported_q),
on: ^join_on,
select: %{}
)
|> select_joined_dimension(dim, query)
|> select_joined_metrics(metrics)
|> apply_order_by(query, metrics)
end
def merge_imported(q, site, %Query{dimensions: []} = query, metrics) do def merge_imported(q, site, %Query{dimensions: []} = query, metrics) do
imported_q = imported_q =
site site
@ -353,51 +282,82 @@ defmodule Plausible.Stats.Imported do
|> select_joined_metrics(metrics) |> select_joined_metrics(metrics)
end end
def merge_imported(q, _, _, _), do: q def merge_imported(q, site, %Query{dimensions: ["event:goal"]} = query, metrics) do
{events, page_regexes} = Filters.Utils.split_goals_query_expressions(query.preloaded_goals)
def merge_imported_pageview_goals(q, _, %Query{include_imported: false}, _, _), do: q Imported.Base.decide_tables(query)
|> Enum.map(fn
"imported_custom_events" ->
Imported.Base.query_imported("imported_custom_events", site, query)
|> where([i], i.visitors > 0)
|> select_merge([i], %{
dim0:
selected_as(
fragment("-indexOf(?, ?)", type(^events, {:array, :string}), i.name),
:dim0
)
})
|> select_imported_metrics(metrics)
|> group_by([], selected_as(:dim0))
|> where([], selected_as(:dim0) != 0)
def merge_imported_pageview_goals(q, site, query, page_exprs, metrics) do "imported_pages" ->
if Imported.Base.decide_table(query) == "imported_pages" do Imported.Base.query_imported("imported_pages", site, query)
page_regexes = Enum.map(page_exprs, &Base.page_regex/1)
imported_q =
"imported_pages"
|> Imported.Base.query_imported(site, query)
|> where([i], i.visitors > 0) |> where([i], i.visitors > 0)
|> where( |> where(
[i], [i],
fragment("notEmpty(multiMatchAllIndices(?, ?) as indices)", i.page, ^page_regexes) fragment(
"notEmpty(multiMatchAllIndices(?, ?) as indices)",
i.page,
type(^page_regexes, {:array, :string})
)
) )
|> join(:array, index in fragment("indices")) |> join(:array, index in fragment("indices"))
|> group_by([_i, index], index) |> group_by([_i, index], index)
|> select_merge([_i, index], %{ |> select_merge([_i, index], %{
name: fragment("concat('Visit ', ?[?])", ^page_exprs, index) dim0: selected_as(type(fragment("?", index), :integer), :dim0)
}) })
|> select_imported_metrics(metrics) |> select_imported_metrics(metrics)
end)
|> Enum.reduce(q, fn imports_q, q ->
naive_dimension_join(q, imports_q, metrics)
end)
end
from(s in Ecto.Query.subquery(q), def merge_imported(q, site, %Query{dimensions: dimensions} = query, metrics) do
if merge_imported_dimensions?(dimensions) do
imported_q =
site
|> Imported.Base.query_imported(query)
|> where([i], i.visitors > 0)
|> group_imported_by(query)
|> select_imported_metrics(metrics)
from(s in subquery(q),
full_join: i in subquery(imported_q), full_join: i in subquery(imported_q),
on: s.name == i.name, on: ^QueryBuilder.build_group_by_join(query),
select: %{} select: %{}
) )
|> select_joined_dimension(:name, query) |> select_joined_dimensions(query)
|> select_joined_metrics(metrics) |> select_joined_metrics(metrics)
else else
q q
end end
end end
def merge_imported(q, _, _, _), do: q
defp merge_imported_dimensions?(dimensions) do
dimensions in [["visit:browser", "visit:browser_version"], ["visit:os", "visit:os_version"]] or
(length(dimensions) == 1 and hd(dimensions) in @imported_dimensions)
end
def total_imported_visitors(site, query) do def total_imported_visitors(site, query) do
site site
|> Imported.Base.query_imported(query) |> Imported.Base.query_imported(query)
|> select_merge([i], %{total_visitors: fragment("sum(?)", i.visitors)}) |> select_merge([i], %{total_visitors: fragment("sum(?)", i.visitors)})
end end
# :TRICKY: Handle backwards compatibility with old breakdown module
defp shortname(query, _dim) when query.v2, do: :dim0
defp shortname(_query, dim), do: dim
defp select_imported_metrics(q, []), do: q defp select_imported_metrics(q, []), do: q
defp select_imported_metrics(q, [:visitors | rest]) do defp select_imported_metrics(q, [:visitors | rest]) do
@ -592,190 +552,175 @@ defmodule Plausible.Stats.Imported do
|> select_imported_metrics(rest) |> select_imported_metrics(rest)
end end
defp group_imported_by(q, dim, query) when dim in [:source, :referrer] do defp group_imported_by(q, query) do
Enum.reduce(query.dimensions, q, fn dimension, q ->
dim = Plausible.Stats.Filters.without_prefix(dimension)
group_imported_by(q, dim, shortname(query, dimension))
end)
end
defp group_imported_by(q, dim, key) when dim in [:source, :referrer] do
q q
|> group_by([i], field(i, ^dim)) |> group_by([i], field(i, ^dim))
|> select_merge([i], %{ |> select_merge([i], %{
^shortname(query, dim) => ^key =>
fragment( selected_as(
"if(empty(?), ?, ?)", fragment(
field(i, ^dim), "if(empty(?), ?, ?)",
@no_ref, field(i, ^dim),
field(i, ^dim) @no_ref,
field(i, ^dim)
),
^key
) )
}) })
end end
defp group_imported_by(q, dim, query) defp group_imported_by(q, dim, key)
when dim in [:utm_source, :utm_medium, :utm_campaign, :utm_term, :utm_content] do when dim in [:utm_source, :utm_medium, :utm_campaign, :utm_term, :utm_content] do
q q
|> group_by([i], field(i, ^dim)) |> group_by([i], field(i, ^dim))
|> where([i], fragment("not empty(?)", field(i, ^dim))) |> where([i], fragment("not empty(?)", field(i, ^dim)))
|> select_merge([i], %{^shortname(query, dim) => field(i, ^dim)}) |> select_merge([i], %{^key => selected_as(field(i, ^dim), ^key)})
end end
defp group_imported_by(q, :page, query) do defp group_imported_by(q, :page, key) do
q q
|> group_by([i], i.page) |> group_by([i], i.page)
|> select_merge([i], %{^shortname(query, :page) => i.page, time_on_page: sum(i.time_on_page)}) |> select_merge([i], %{^key => selected_as(i.page, ^key), time_on_page: sum(i.time_on_page)})
end end
defp group_imported_by(q, :country, query) do defp group_imported_by(q, :country, key) do
q q
|> group_by([i], i.country) |> group_by([i], i.country)
|> where([i], i.country != "ZZ") |> where([i], i.country != "ZZ")
|> select_merge([i], %{^shortname(query, :country) => i.country}) |> select_merge([i], %{^key => selected_as(i.country, ^key)})
end end
defp group_imported_by(q, :region, query) do defp group_imported_by(q, :region, key) do
q q
|> group_by([i], i.region) |> group_by([i], i.region)
|> where([i], i.region != "") |> where([i], i.region != "")
|> select_merge([i], %{^shortname(query, :region) => i.region}) |> select_merge([i], %{^key => selected_as(i.region, ^key)})
end end
defp group_imported_by(q, :city, query) do defp group_imported_by(q, :city, key) do
q q
|> group_by([i], i.city) |> group_by([i], i.city)
|> where([i], i.city != 0 and not is_nil(i.city)) |> where([i], i.city != 0 and not is_nil(i.city))
|> select_merge([i], %{^shortname(query, :city) => i.city}) |> select_merge([i], %{^key => selected_as(i.city, ^key)})
end end
defp group_imported_by(q, dim, query) when dim in [:device, :browser] do defp group_imported_by(q, dim, key) when dim in [:device, :browser] do
q q
|> group_by([i], field(i, ^dim)) |> group_by([i], field(i, ^dim))
|> select_merge([i], %{ |> select_merge([i], %{
^shortname(query, dim) => ^key =>
fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim)) selected_as(
}) fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim)),
end ^key
defp group_imported_by(q, :browser_version, query) do
q
|> group_by([i], [i.browser, i.browser_version])
|> select_merge([i], %{
^shortname(query, :browser) =>
fragment("if(empty(?), ?, ?)", i.browser, @not_set, i.browser),
^shortname(query, :browser_version) =>
fragment(
"if(empty(?), ?, ?)",
i.browser_version,
@not_set,
i.browser_version
) )
}) })
end end
defp group_imported_by(q, :os, query) do defp group_imported_by(q, :browser_version, key) do
q
|> group_by([i], [i.browser_version])
|> select_merge([i], %{
^key =>
selected_as(
fragment(
"if(empty(?), ?, ?)",
i.browser_version,
@not_set,
i.browser_version
),
^key
)
})
end
defp group_imported_by(q, :os, key) do
q q
|> group_by([i], i.operating_system) |> group_by([i], i.operating_system)
|> select_merge([i], %{ |> select_merge([i], %{
^shortname(query, :os) => ^key =>
fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system) selected_as(
}) fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system),
end ^key
defp group_imported_by(q, :os_version, query) do
q
|> group_by([i], [i.operating_system, i.operating_system_version])
|> select_merge([i], %{
^shortname(query, :os) =>
fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system),
^shortname(query, :os_version) =>
fragment(
"if(empty(?), ?, ?)",
i.operating_system_version,
@not_set,
i.operating_system_version
) )
}) })
end end
defp group_imported_by(q, dim, query) when dim in [:entry_page, :exit_page] do defp group_imported_by(q, :os_version, key) do
q
|> group_by([i], [i.operating_system_version])
|> select_merge([i], %{
^key =>
selected_as(
fragment(
"if(empty(?), ?, ?)",
i.operating_system_version,
@not_set,
i.operating_system_version
),
^key
)
})
end
defp group_imported_by(q, dim, key) when dim in [:entry_page, :exit_page] do
q q
|> group_by([i], field(i, ^dim)) |> group_by([i], field(i, ^dim))
|> select_merge([i], %{^shortname(query, dim) => field(i, ^dim)}) |> select_merge([i], %{^key => selected_as(field(i, ^dim), ^key)})
end end
defp group_imported_by(q, :name, query) do defp group_imported_by(q, :name, key) do
q q
|> group_by([i], i.name) |> group_by([i], i.name)
|> select_merge([i], %{^shortname(query, :name) => i.name}) |> select_merge([i], %{^key => selected_as(i.name, ^key)})
end end
defp group_imported_by(q, :url, query) when query.v2 do defp group_imported_by(q, :url, key) do
q q
|> group_by([i], i.link_url) |> group_by([i], i.link_url)
|> select_merge([i], %{ |> select_merge([i], %{
^shortname(query, :url) => fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none) ^key => selected_as(fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none), ^key)
}) })
end end
defp group_imported_by(q, :url, _query) do defp group_imported_by(q, :path, key) do
q
|> group_by([i], i.link_url)
|> select_merge([i], %{
breakdown_prop_value: fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)
})
end
defp group_imported_by(q, :path, query) when query.v2 do
q q
|> group_by([i], i.path) |> group_by([i], i.path)
|> select_merge([i], %{ |> select_merge([i], %{
^shortname(query, :path) => fragment("if(not empty(?), ?, ?)", i.path, i.path, @none) ^key => selected_as(fragment("if(not empty(?), ?, ?)", i.path, i.path, @none), ^key)
}) })
end end
defp group_imported_by(q, :path, _query) do defp select_joined_dimensions(q, query) do
q Enum.reduce(query.dimensions, q, fn dimension, q ->
|> group_by([i], i.path) select_joined_dimension(q, dimension, shortname(query, dimension))
|> select_merge([i], %{ end)
breakdown_prop_value: fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)
})
end end
defp select_joined_dimension(q, :city, query) do defp select_joined_dimension(q, "visit:city", key) do
select_merge(q, [s, i], %{ select_merge(q, [s, i], %{
^shortname(query, :city) => fragment("greatest(?,?)", i.city, s.city) ^key => selected_as(fragment("greatest(?,?)", field(i, ^key), field(s, ^key)), ^key)
}) })
end end
defp select_joined_dimension(q, :os_version, query) when not query.v2 do defp select_joined_dimension(q, _dimension, key) do
select_merge(q, [s, i], %{ select_merge(q, [s, i], %{
os: fragment("if(empty(?), ?, ?)", s.os, i.os, s.os), ^key =>
os_version: fragment("if(empty(?), ?, ?)", s.os_version, i.os_version, s.os_version) selected_as(
}) fragment(
end "if(empty(?), ?, ?)",
field(s, ^key),
defp select_joined_dimension(q, :browser_version, query) when not query.v2 do field(i, ^key),
select_merge(q, [s, i], %{ field(s, ^key)
browser: fragment("if(empty(?), ?, ?)", s.browser, i.browser, s.browser), ),
browser_version: ^key
fragment("if(empty(?), ?, ?)", s.browser_version, i.browser_version, s.browser_version)
})
end
defp select_joined_dimension(q, dim, query) when dim in [:url, :path] and not query.v2 do
select_merge(q, [s, i], %{
breakdown_prop_value:
fragment(
"if(empty(?), ?, ?)",
s.breakdown_prop_value,
i.breakdown_prop_value,
s.breakdown_prop_value
)
})
end
defp select_joined_dimension(q, dim, query) do
select_merge(q, [s, i], %{
^shortname(query, dim) =>
fragment(
"if(empty(?), ?, ?)",
field(s, ^shortname(query, dim)),
field(i, ^shortname(query, dim)),
field(s, ^shortname(query, dim))
) )
}) })
end end
@ -882,21 +827,12 @@ defmodule Plausible.Stats.Imported do
|> select_joined_metrics(rest) |> select_joined_metrics(rest)
end end
defp apply_order_by(q, %Query{v2: true}, _), do: q
defp apply_order_by(q, query, [:visitors | rest]) do
order_by(q, [s, i], desc: s.visitors + i.visitors)
|> apply_order_by(query, rest)
end
defp apply_order_by(q, _query, _), do: q
defp naive_dimension_join(q1, q2, metrics) do defp naive_dimension_join(q1, q2, metrics) do
from(a in Ecto.Query.subquery(q1), from(a in subquery(q1),
full_join: b in subquery(q2), full_join: b in subquery(q2),
on: a.dim0 == b.dim0, on: a.dim0 == b.dim0,
select: %{ select: %{
dim0: fragment("coalesce(?, ?)", a.dim0, b.dim0) dim0: selected_as(fragment("if(? != 0, ?, ?)", a.dim0, a.dim0, b.dim0), :dim0)
} }
) )
|> select_joined_metrics(metrics) |> select_joined_metrics(metrics)

7
lib/plausible/stats/metrics.ex
View File

@ -17,7 +17,8 @@ defmodule Plausible.Stats.Metrics do
:events, :events,
:conversion_rate, :conversion_rate,
:group_conversion_rate, :group_conversion_rate,
:time_on_page :time_on_page,
:percentage
] ++ on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: []) ] ++ on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: [])
@metric_mappings Enum.into(@all_metrics, %{}, fn metric -> {to_string(metric), metric} end) @metric_mappings Enum.into(@all_metrics, %{}, fn metric -> {to_string(metric), metric} end)
@ -27,4 +28,8 @@ defmodule Plausible.Stats.Metrics do
def from_string!(str) do def from_string!(str) do
Map.fetch!(@metric_mappings, str) Map.fetch!(@metric_mappings, str)
end end
def from_string(str) do
Map.fetch(@metric_mappings, str)
end
end end

13
lib/plausible/stats/query.ex
View File

@ -18,7 +18,12 @@ defmodule Plausible.Stats.Query do
order_by: nil, order_by: nil,
timezone: nil, timezone: nil,
v2: false, v2: false,
preloaded_goals: [] legacy_breakdown: false,
preloaded_goals: [],
include: %{
imports: false,
time_labels: false
}
require OpenTelemetry.Tracer, as: Tracer require OpenTelemetry.Tracer, as: Tracer
alias Plausible.Stats.{Filters, Interval, Imported} alias Plausible.Stats.{Filters, Interval, Imported}
@ -225,6 +230,12 @@ defmodule Plausible.Stats.Query do
struct!(query, filters: Filters.parse(params["filters"])) struct!(query, filters: Filters.parse(params["filters"]))
end end
def set(query, keywords) do
query
|> struct!(keywords)
|> refresh_imported_opts()
end
@spec set_dimensions(t(), list(String.t())) :: t() @spec set_dimensions(t(), list(String.t())) :: t()
def set_dimensions(query, dimensions) do def set_dimensions(query, dimensions) do
query query

16
lib/plausible/stats/query_optimizer.ex
View File

@ -3,6 +3,7 @@ defmodule Plausible.Stats.QueryOptimizer do
Methods to manipulate Query for business logic reasons before building an ecto query. Methods to manipulate Query for business logic reasons before building an ecto query.
""" """
use Plausible
alias Plausible.Stats.{Query, TableDecider, Util} alias Plausible.Stats.{Query, TableDecider, Util}
@doc """ @doc """
@ -148,8 +149,17 @@ defmodule Plausible.Stats.QueryOptimizer do
dimension -> dimension dimension -> dimension
end) end)
query filters =
|> Query.set_metrics(session_metrics) if "event:page" in query.dimensions do
|> Query.set_dimensions(dimensions) query.filters
|> Enum.map(fn
[op, "event:page" | rest] -> [op, "visit:entry_page" | rest]
filter -> filter
end)
else
query.filters
end
Query.set(query, filters: filters, metrics: session_metrics, dimensions: dimensions)
end end
end end

141
lib/plausible/stats/sql/expression.ex
View File

@ -11,84 +11,121 @@ defmodule Plausible.Stats.SQL.Expression do
@no_ref "Direct / None" @no_ref "Direct / None"
@not_set "(not set)" @not_set "(not set)"
defmacrop field_or_blank_value(expr, empty_value) do defmacrop field_or_blank_value(expr, empty_value, select_alias) do
quote do quote do
dynamic( dynamic(
[t], [t],
fragment("if(empty(?), ?, ?)", unquote(expr), unquote(empty_value), unquote(expr)) selected_as(
fragment("if(empty(?), ?, ?)", unquote(expr), unquote(empty_value), unquote(expr)),
^unquote(select_alias)
)
) )
end end
end end
def dimension("time:hour", query) do def dimension("time:hour", query, select_alias) do
dynamic([t], fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone))
end
def dimension("time:day", query) do
dynamic([t], fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone))
end
def dimension("time:month", query) do
dynamic([t], fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone))
end
def dimension("event:name", _query), do: dynamic([t], t.name)
def dimension("event:page", _query), do: dynamic([t], t.pathname)
def dimension("event:hostname", _query), do: dynamic([t], t.hostname)
def dimension("event:props:" <> property_name, _query) do
dynamic( dynamic(
[t], [t],
fragment( selected_as(
"if(not empty(?), ?, '(none)')", fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone),
get_by_key(t, :meta, ^property_name), ^select_alias
get_by_key(t, :meta, ^property_name)
) )
) )
end end
def dimension("visit:entry_page", _query), do: dynamic([t], t.entry_page) def dimension("time:day", query, select_alias) do
def dimension("visit:exit_page", _query), do: dynamic([t], t.exit_page) dynamic(
[t],
selected_as(
fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone),
^select_alias
)
)
end
def dimension("visit:utm_medium", _query), def dimension("time:month", query, select_alias) do
do: field_or_blank_value(t.utm_medium, @not_set) dynamic(
[t],
selected_as(
fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone),
^select_alias
)
)
end
def dimension("visit:utm_source", _query), def dimension("event:name", _query, select_alias),
do: field_or_blank_value(t.utm_source, @not_set) do: dynamic([t], selected_as(t.name, ^select_alias))
def dimension("visit:utm_campaign", _query), def dimension("event:page", _query, select_alias),
do: field_or_blank_value(t.utm_campaign, @not_set) do: dynamic([t], selected_as(t.pathname, ^select_alias))
def dimension("visit:utm_content", _query), def dimension("event:hostname", _query, select_alias),
do: field_or_blank_value(t.utm_content, @not_set) do: dynamic([t], selected_as(t.hostname, ^select_alias))
def dimension("visit:utm_term", _query), def dimension("event:props:" <> property_name, _query, select_alias) do
do: field_or_blank_value(t.utm_term, @not_set) dynamic(
[t],
selected_as(
fragment(
"if(not empty(?), ?, '(none)')",
get_by_key(t, :meta, ^property_name),
get_by_key(t, :meta, ^property_name)
),
^select_alias
)
)
end
def dimension("visit:source", _query), def dimension("visit:entry_page", _query, select_alias),
do: field_or_blank_value(t.source, @no_ref) do: dynamic([t], selected_as(t.entry_page, ^select_alias))
def dimension("visit:referrer", _query), def dimension("visit:exit_page", _query, select_alias),
do: field_or_blank_value(t.referrer, @no_ref) do: dynamic([t], selected_as(t.exit_page, ^select_alias))
def dimension("visit:device", _query), def dimension("visit:utm_medium", _query, select_alias),
do: field_or_blank_value(t.device, @not_set) do: field_or_blank_value(t.utm_medium, @not_set, select_alias)
def dimension("visit:os", _query), do: field_or_blank_value(t.os, @not_set) def dimension("visit:utm_source", _query, select_alias),
do: field_or_blank_value(t.utm_source, @not_set, select_alias)
def dimension("visit:os_version", _query), def dimension("visit:utm_campaign", _query, select_alias),
do: field_or_blank_value(t.os_version, @not_set) do: field_or_blank_value(t.utm_campaign, @not_set, select_alias)
def dimension("visit:browser", _query), def dimension("visit:utm_content", _query, select_alias),
do: field_or_blank_value(t.browser, @not_set) do: field_or_blank_value(t.utm_content, @not_set, select_alias)
def dimension("visit:browser_version", _query), def dimension("visit:utm_term", _query, select_alias),
do: field_or_blank_value(t.browser_version, @not_set) do: field_or_blank_value(t.utm_term, @not_set, select_alias)
# :TODO: Locations also set extra filters def dimension("visit:source", _query, select_alias),
def dimension("visit:country", _query), do: dynamic([t], t.country) do: field_or_blank_value(t.source, @no_ref, select_alias)
def dimension("visit:region", _query), do: dynamic([t], t.region)
def dimension("visit:city", _query), do: dynamic([t], t.city) def dimension("visit:referrer", _query, select_alias),
do: field_or_blank_value(t.referrer, @no_ref, select_alias)
def dimension("visit:device", _query, select_alias),
do: field_or_blank_value(t.device, @not_set, select_alias)
def dimension("visit:os", _query, select_alias),
do: field_or_blank_value(t.os, @not_set, select_alias)
def dimension("visit:os_version", _query, select_alias),
do: field_or_blank_value(t.os_version, @not_set, select_alias)
def dimension("visit:browser", _query, select_alias),
do: field_or_blank_value(t.browser, @not_set, select_alias)
def dimension("visit:browser_version", _query, select_alias),
do: field_or_blank_value(t.browser_version, @not_set, select_alias)
def dimension("visit:country", _query, select_alias),
do: dynamic([t], selected_as(t.country, ^select_alias))
def dimension("visit:region", _query, select_alias),
do: dynamic([t], selected_as(t.region, ^select_alias))
def dimension("visit:city", _query, select_alias),
do: dynamic([t], selected_as(t.city, ^select_alias))
defmacro event_goal_join(events, page_regexes) do defmacro event_goal_join(events, page_regexes) do
quote do quote do

47
lib/plausible/stats/sql/query_builder.ex
View File

@ -7,7 +7,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
import Plausible.Stats.Imported import Plausible.Stats.Imported
import Plausible.Stats.Util import Plausible.Stats.Util
alias Plausible.Stats.{Base, Query, QueryOptimizer, TableDecider, Filters, Metrics} alias Plausible.Stats.{Base, Query, QueryOptimizer, TableDecider, Filters}
alias Plausible.Stats.SQL.Expression alias Plausible.Stats.SQL.Expression
require Plausible.Stats.SQL.Expression require Plausible.Stats.SQL.Expression
@ -44,6 +44,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
|> merge_imported(site, events_query, events_query.metrics) |> merge_imported(site, events_query, events_query.metrics)
|> maybe_add_global_conversion_rate(site, events_query) |> maybe_add_global_conversion_rate(site, events_query)
|> maybe_add_group_conversion_rate(site, events_query) |> maybe_add_group_conversion_rate(site, events_query)
|> Base.add_percentage_metric(site, events_query, events_query.metrics)
end end
defp join_sessions_if_needed(q, site, query) do defp join_sessions_if_needed(q, site, query) do
@ -84,6 +85,9 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
|> join_events_if_needed(site, sessions_query) |> join_events_if_needed(site, sessions_query)
|> build_group_by(sessions_query) |> build_group_by(sessions_query)
|> merge_imported(site, sessions_query, sessions_query.metrics) |> merge_imported(site, sessions_query, sessions_query.metrics)
|> maybe_add_global_conversion_rate(site, sessions_query)
|> maybe_add_group_conversion_rate(site, sessions_query)
|> Base.add_percentage_metric(site, sessions_query, sessions_query.metrics)
end end
def join_events_if_needed(q, site, query) do def join_events_if_needed(q, site, query) do
@ -123,36 +127,23 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
^shortname(query, dimension) => fragment("?", goal) ^shortname(query, dimension) => fragment("?", goal)
}, },
group_by: goal, group_by: goal,
where: goal != 0 where: goal != 0 and (e.name == "pageview" or goal < 0)
) )
end end
defp dimension_group_by(q, query, dimension) do defp dimension_group_by(q, query, dimension) do
key = shortname(query, dimension)
q q
|> select_merge(^%{shortname(query, dimension) => Expression.dimension(dimension, query)}) |> select_merge(^%{key => Expression.dimension(dimension, query, key)})
|> group_by(^Expression.dimension(dimension, query)) |> group_by([], selected_as(^key))
end end
defp build_order_by(q, query, mode) do defp build_order_by(q, query) do
Enum.reduce(query.order_by || [], q, &build_order_by(&2, query, &1, mode)) Enum.reduce(query.order_by || [], q, &build_order_by(&2, query, &1))
end end
def build_order_by(q, query, {metric_or_dimension, order_direction}, :inner) do def build_order_by(q, query, {metric_or_dimension, order_direction}) do
order_by(
q,
[t],
^{
order_direction,
if(
Metrics.metric?(metric_or_dimension),
do: dynamic([], selected_as(^shortname(query, metric_or_dimension))),
else: Expression.dimension(metric_or_dimension, query)
)
}
)
end
def build_order_by(q, query, {metric_or_dimension, order_direction}, :outer) do
order_by( order_by(
q, q,
[t], [t],
@ -262,10 +253,10 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
defp join_query_results({nil, _}, {nil, _}), do: nil defp join_query_results({nil, _}, {nil, _}), do: nil
defp join_query_results({events_q, events_query}, {nil, _}), defp join_query_results({events_q, events_query}, {nil, _}),
do: events_q |> build_order_by(events_query, :inner) do: events_q |> build_order_by(events_query)
defp join_query_results({nil, _}, {sessions_q, sessions_query}), defp join_query_results({nil, events_query}, {sessions_q, _}),
do: sessions_q |> build_order_by(sessions_query, :inner) do: sessions_q |> build_order_by(events_query)
defp join_query_results({events_q, events_query}, {sessions_q, sessions_query}) do defp join_query_results({events_q, events_query}, {sessions_q, sessions_query}) do
join(subquery(events_q), :left, [e], s in subquery(sessions_q), join(subquery(events_q), :left, [e], s in subquery(sessions_q),
@ -274,12 +265,12 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
|> select_join_fields(events_query, events_query.dimensions, e) |> select_join_fields(events_query, events_query.dimensions, e)
|> select_join_fields(events_query, events_query.metrics, e) |> select_join_fields(events_query, events_query.metrics, e)
|> select_join_fields(sessions_query, List.delete(sessions_query.metrics, :sample_percent), s) |> select_join_fields(sessions_query, List.delete(sessions_query.metrics, :sample_percent), s)
|> build_order_by(events_query, :outer) |> build_order_by(events_query)
end end
defp build_group_by_join(%Query{dimensions: []}), do: true def build_group_by_join(%Query{dimensions: []}), do: true
defp build_group_by_join(query) do def build_group_by_join(query) do
query.dimensions query.dimensions
|> Enum.map(fn dim -> |> Enum.map(fn dim ->
dynamic([e, s], field(e, ^shortname(query, dim)) == field(s, ^shortname(query, dim))) dynamic([e, s], field(e, ^shortname(query, dim)) == field(s, ^shortname(query, dim)))

31
lib/plausible/stats/table_decider.ex
View File

@ -29,34 +29,45 @@ defmodule Plausible.Stats.TableDecider do
|> filter_keys() |> filter_keys()
|> partition(query, &filters_partitioner/2) |> partition(query, &filters_partitioner/2)
%{event: event_only_dimensions, session: session_only_dimensions} =
partition(query.dimensions, query, &filters_partitioner/2)
cond do cond do
# Only one table needs to be queried # Only one table needs to be queried
empty?(event_only_metrics) && empty?(event_only_filters) -> empty?(event_only_metrics) && empty?(event_only_filters) && empty?(event_only_dimensions) ->
{[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics} {[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}
empty?(session_only_metrics) && empty?(session_only_filters) -> empty?(session_only_metrics) && empty?(session_only_filters) &&
empty?(session_only_dimensions) ->
{event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics} {event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics}
# Filters on both events and sessions, but only one kind of metric # Filters and/or dimensions on both events and sessions, but only one kind of metric
empty?(event_only_metrics) -> empty?(event_only_metrics) && empty?(event_only_dimensions) ->
{[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics} {[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}
empty?(session_only_metrics) -> empty?(session_only_metrics) && empty?(session_only_dimensions) ->
{event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics} {event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics}
# Default: prefer sessions # Default: prefer events
true -> true ->
{event_only_metrics ++ sample_percent, {event_only_metrics ++ either_metrics ++ sample_percent,
session_only_metrics ++ either_metrics ++ sample_percent, other_metrics} session_only_metrics ++ sample_percent, other_metrics}
end end
end end
defp filter_keys(query) do defp filter_keys(query) do
query.filters query.filters
|> Enum.map(fn [_, filter_key | _rest] -> filter_key end) |> Enum.map(fn [_, filter_key | _rest] -> filter_key end)
|> Enum.concat(query.dimensions)
end end
defp metric_partitioner(%Query{v2: true}, :conversion_rate), do: :either
defp metric_partitioner(%Query{v2: true}, :group_conversion_rate), do: :either
defp metric_partitioner(%Query{v2: true}, :visitors), do: :either
defp metric_partitioner(%Query{v2: true}, :visits), do: :either
# Note: This is inaccurate when filtering but required for old backwards compatibility
defp metric_partitioner(%Query{legacy_breakdown: true}, :pageviews), do: :either
defp metric_partitioner(%Query{legacy_breakdown: true}, :events), do: :either
defp metric_partitioner(_, :conversion_rate), do: :event defp metric_partitioner(_, :conversion_rate), do: :event
defp metric_partitioner(_, :group_conversion_rate), do: :event defp metric_partitioner(_, :group_conversion_rate), do: :event
defp metric_partitioner(_, :average_revenue), do: :event defp metric_partitioner(_, :average_revenue), do: :event
@ -76,7 +87,7 @@ defmodule Plausible.Stats.TableDecider do
# Calculated metrics - handled on callsite separately from other metrics. # Calculated metrics - handled on callsite separately from other metrics.
defp metric_partitioner(_, :time_on_page), do: :other defp metric_partitioner(_, :time_on_page), do: :other
defp metric_partitioner(_, :total_visitors), do: :other defp metric_partitioner(_, :total_visitors), do: :other
defp metric_partitioner(_, :percentage), do: :other defp metric_partitioner(_, :percentage), do: :either
# Sample percentage is included in both tables if queried. # Sample percentage is included in both tables if queried.
defp metric_partitioner(_, :sample_percent), do: :sample_percent defp metric_partitioner(_, :sample_percent), do: :sample_percent

8
test/plausible/imported/csv_importer_test.exs
View File

@ -622,7 +622,7 @@ defmodule Plausible.Imported.CSVImporterTest do
case params_or_site do case params_or_site do
%Plausible.Site{} = site -> %Plausible.Site{} = site ->
common_params.(site) common_params.(site)
|> Map.put("metrics", "visitors,visits,pageviews,visit_duration,bounce_rate") |> Map.put("metrics", "visitors,visits,visit_duration,bounce_rate")
|> Map.put("limit", 1000) |> Map.put("limit", 1000)
|> Map.put("property", by) |> Map.put("property", by)
@ -669,7 +669,7 @@ defmodule Plausible.Imported.CSVImporterTest do
assert exported["pageviews"] == imported["pageviews"] assert exported["pageviews"] == imported["pageviews"]
assert exported["bounce_rate"] == imported["bounce_rate"] assert exported["bounce_rate"] == imported["bounce_rate"]
assert_in_delta exported["visitors"], imported["visitors"], 1 assert_in_delta exported["visitors"], imported["visitors"], 1
assert exported["visits"] == imported["visits"] assert_in_delta exported["visits"], imported["visits"], 1
assert_in_delta exported["visit_duration"], imported["visit_duration"], 1 assert_in_delta exported["visit_duration"], imported["visit_duration"], 1
end) end)
@ -810,7 +810,7 @@ defmodule Plausible.Imported.CSVImporterTest do
_no_diff = 0 _no_diff = 0
end end
end) end)
) == [0.0, 0.0, 0.0, 0.0, 0.03614457831325302] ) == [0.0, 0.0, 0.0, 0.0, 0.0]
# NOTE: city breakdown's visit duration difference is up to 14% # NOTE: city breakdown's visit duration difference is up to 14%
assert summary(field(exported_cities, "visit_duration")) == [0, 0.0, 0.0, 1.0, 1718] assert summary(field(exported_cities, "visit_duration")) == [0, 0.0, 0.0, 1.0, 1718]
@ -829,7 +829,7 @@ defmodule Plausible.Imported.CSVImporterTest do
_no_diff = 0 _no_diff = 0
end end
end) end)
) == [0, 0.0, 0.0, 0.0, 0.1428571428571429] ) == [0, 0.0, 0.0, 0.0, 0.0]
# NOTE: city breakdown's visitors relative difference is up to 27% # NOTE: city breakdown's visitors relative difference is up to 27%
assert summary(field(exported_cities, "visitors")) == [1, 1.0, 1.0, 2.0, 22] assert summary(field(exported_cities, "visitors")) == [1, 1.0, 1.0, 2.0, 22]

60
test/plausible/imported/google_analytics4_test.exs
View File

@ -117,7 +117,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
breakdown_params = breakdown_params =
common_params common_params
|> Map.put("metrics", "visitors,visits,pageviews,visit_duration,bounce_rate") |> Map.put("metrics", "visitors,visits,visit_duration,bounce_rate")
|> Map.put("limit", 1000) |> Map.put("limit", 1000)
%{key: api_key} = insert(:api_key, user: user) %{key: api_key} = insert(:api_key, user: user)
@ -467,7 +467,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.first(results) == %{ assert List.first(results) == %{
"bounce_rate" => 35.0, "bounce_rate" => 35.0,
"pageviews" => 6229,
"visit_duration" => 40.0, "visit_duration" => 40.0,
"visitors" => 4671, "visitors" => 4671,
"visits" => 4917, "visits" => 4917,
@ -475,12 +474,11 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
} }
assert List.last(results) == %{ assert List.last(results) == %{
"bounce_rate" => 100.0, "bounce_rate" => 0.0,
"pageviews" => 1, "source" => "yahoo",
"visit_duration" => 0.0, "visit_duration" => 41.0,
"visitors" => 1, "visitors" => 1,
"visits" => 1, "visits" => 1
"source" => "petalsearch.com"
} }
end end
@ -493,7 +491,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert [ assert [
%{ %{
"bounce_rate" => 35.0, "bounce_rate" => 35.0,
"pageviews" => 6399,
"utm_medium" => "organic", "utm_medium" => "organic",
"visit_duration" => 40.0, "visit_duration" => 40.0,
"visitors" => 4787, "visitors" => 4787,
@ -501,7 +498,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
}, },
%{ %{
"bounce_rate" => 58.0, "bounce_rate" => 58.0,
"pageviews" => 491,
"utm_medium" => "referral", "utm_medium" => "referral",
"visit_duration" => 27.0, "visit_duration" => 27.0,
"visitors" => 294, "visitors" => 294,
@ -520,7 +516,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.first(results) == %{ assert List.first(results) == %{
"bounce_rate" => 35.0, "bounce_rate" => 35.0,
"pageviews" => 838,
"visit_duration" => 43.0, "visit_duration" => 43.0,
"visitors" => 675, "visitors" => 675,
"visits" => 712, "visits" => 712,
@ -529,11 +524,10 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.last(results) == %{ assert List.last(results) == %{
"bounce_rate" => 0.0, "bounce_rate" => 0.0,
"pageviews" => 1, "entry_page" => "/znamenitosti-rima-koje-treba-vidjeti",
"visit_duration" => 27.0, "visit_duration" => 40.0,
"visitors" => 1, "visitors" => 1,
"visits" => 1, "visits" => 1
"entry_page" => "/kad-lisce-pada"
} }
end end
@ -543,12 +537,11 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
%{"results" => results} = %{"results" => results} =
get(conn, "/api/v1/stats/breakdown", params) |> json_response(200) get(conn, "/api/v1/stats/breakdown", params) |> json_response(200)
assert length(results) == 488 assert length(results) == 494
assert List.first(results) == %{ assert List.first(results) == %{
"bounce_rate" => 35.0, "bounce_rate" => 35.0,
"city" => 792_680, "city" => 792_680,
"pageviews" => 1650,
"visit_duration" => 39.0, "visit_duration" => 39.0,
"visitors" => 1233, "visitors" => 1233,
"visits" => 1273 "visits" => 1273
@ -556,9 +549,8 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.last(results) == %{ assert List.last(results) == %{
"bounce_rate" => 0.0, "bounce_rate" => 0.0,
"city" => 4_399_605, "city" => 11_951_298,
"pageviews" => 7, "visit_duration" => 271.0,
"visit_duration" => 128.0,
"visitors" => 1, "visitors" => 1,
"visits" => 1 "visits" => 1
} }
@ -574,7 +566,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.first(results) == %{ assert List.first(results) == %{
"bounce_rate" => 38.0, "bounce_rate" => 38.0,
"pageviews" => 7041,
"visit_duration" => 37.0, "visit_duration" => 37.0,
"visitors" => 5277, "visitors" => 5277,
"visits" => 5532, "visits" => 5532,
@ -583,7 +574,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.last(results) == %{ assert List.last(results) == %{
"bounce_rate" => 37.0, "bounce_rate" => 37.0,
"pageviews" => 143,
"visit_duration" => 60.0, "visit_duration" => 60.0,
"visitors" => 97, "visitors" => 97,
"visits" => 100, "visits" => 100,
@ -601,7 +591,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.first(results) == %{ assert List.first(results) == %{
"bounce_rate" => 33.0, "bounce_rate" => 33.0,
"pageviews" => 8143,
"visit_duration" => 50.0, "visit_duration" => 50.0,
"visitors" => 4625, "visitors" => 4625,
"visits" => 4655, "visits" => 4655,
@ -610,7 +599,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.last(results) == %{ assert List.last(results) == %{
"bounce_rate" => 0.0, "bounce_rate" => 0.0,
"pageviews" => 6,
"visit_duration" => 0.0, "visit_duration" => 0.0,
"visitors" => 1, "visitors" => 1,
"visits" => 1, "visits" => 1,
@ -628,7 +616,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.first(results) == %{ assert List.first(results) == %{
"bounce_rate" => 34.0, "bounce_rate" => 34.0,
"pageviews" => 5827,
"visit_duration" => 41.0, "visit_duration" => 41.0,
"visitors" => 4319, "visitors" => 4319,
"visits" => 4495, "visits" => 4495,
@ -637,7 +624,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.last(results) == %{ assert List.last(results) == %{
"bounce_rate" => 0.0, "bounce_rate" => 0.0,
"pageviews" => 6,
"visit_duration" => 0.0, "visit_duration" => 0.0,
"visitors" => 1, "visitors" => 1,
"visits" => 1, "visits" => 1,
@ -657,7 +643,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
"bounce_rate" => 32.0, "bounce_rate" => 32.0,
"os" => "Android", "os" => "Android",
"os_version" => "13.0.0", "os_version" => "13.0.0",
"pageviews" => 1673,
"visit_duration" => 42.0, "visit_duration" => 42.0,
"visitors" => 1247, "visitors" => 1247,
"visits" => 1295 "visits" => 1295
@ -665,17 +650,16 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert List.last(results) == %{ assert List.last(results) == %{
"bounce_rate" => 0.0, "bounce_rate" => 0.0,
"os" => "iOS", "os" => "Chrome OS",
"os_version" => "15.1", "os_version" => "x86_64 15662.76.0",
"pageviews" => 1, "visit_duration" => 16.0,
"visit_duration" => 54.0,
"visitors" => 1, "visitors" => 1,
"visits" => 1 "visits" => 1
} }
end end
defp assert_pages(conn, params) do defp assert_pages(conn, params) do
metrics = "visitors,visits,pageviews,time_on_page,visit_duration,bounce_rate" metrics = "visitors,visits,time_on_page,visit_duration,bounce_rate"
params = params =
params params
@ -686,7 +670,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
%{"results" => results} = %{"results" => results} =
get(conn, "/api/v1/stats/breakdown", params) |> json_response(200) get(conn, "/api/v1/stats/breakdown", params) |> json_response(200)
assert length(results) == 729 assert length(results) == 730
# The `event:page` breakdown is currently using the `entry_page` # The `event:page` breakdown is currently using the `entry_page`
# property to allow querying session metrics. # property to allow querying session metrics.
@ -696,7 +680,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
# it will allow us to assert on the session metrics as well. # it will allow us to assert on the session metrics as well.
assert Enum.at(results, 2) == %{ assert Enum.at(results, 2) == %{
"page" => "/", "page" => "/",
"pageviews" => 5537,
"time_on_page" => 17.677262055264585, "time_on_page" => 17.677262055264585,
"visitors" => 371, "visitors" => 371,
"visits" => 212, "visits" => 212,
@ -707,13 +690,12 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
# This page was never an entry_page in the imported data, and # This page was never an entry_page in the imported data, and
# therefore the session metrics are returned as `nil`. # therefore the session metrics are returned as `nil`.
assert List.last(results) == %{ assert List.last(results) == %{
"page" => "/5-dobrih-razloga-zasto-zapoceti-dan-zobenom-kasom/", "bounce_rate" => 0.0,
"pageviews" => 2, "page" => "/znamenitosti-rima-koje-treba-vidjeti/",
"time_on_page" => 10.0, "time_on_page" => 40.0,
"visit_duration" => 0.0,
"visitors" => 1, "visitors" => 1,
"visits" => 1, "visits" => 1
"bounce_rate" => nil,
"visit_duration" => nil
} }
end end
end end

6
test/plausible/stats/table_decider_test.exs
View File

@ -76,7 +76,7 @@ defmodule Plausible.Stats.TableDeciderTest do
query = make_query(false, []) query = make_query(false, [])
assert partition_metrics([:time_on_page, :percentage, :total_visitors], query) == assert partition_metrics([:time_on_page, :percentage, :total_visitors], query) ==
{[], [], [:time_on_page, :percentage, :total_visitors]} {[], [:percentage], [:time_on_page, :total_visitors]}
end end
test "raises if unknown metric" do test "raises if unknown metric" do
@ -108,11 +108,11 @@ defmodule Plausible.Stats.TableDeciderTest do
{[], [:visit_duration, :visits], []} {[], [:visit_duration, :visits], []}
end end
test "metrics that can be calculated on either are biased to sessions" do test "metrics that can be calculated on either are biased to events" do
query = make_query(true, []) query = make_query(true, [])
assert partition_metrics([:bounce_rate, :total_revenue, :visitors], query) == assert partition_metrics([:bounce_rate, :total_revenue, :visitors], query) ==
{[:total_revenue], [:bounce_rate, :visitors], []} {[:total_revenue, :visitors], [:bounce_rate], []}
end end
test "sample_percent is handled with either metrics" do test "sample_percent is handled with either metrics" do

2
test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv
View File

@ -1,2 +1,2 @@
name,visitors,pageviews,bounce_rate,time_on_page name,visitors,pageviews,bounce_rate,time_on_page
/some-other-page,1,1,,60.0 /some-other-page,1,1,0,60.0

1 name visitors pageviews bounce_rate time_on_page
2 /some-other-page 1 1 0 60.0

2
test/plausible_web/controllers/CSVs/30d/pages.csv
View File

@ -1,4 +1,4 @@
name,visitors,pageviews,bounce_rate,time_on_page name,visitors,pageviews,bounce_rate,time_on_page
/,4,3,67, /,4,3,67,
/signup,1,1,0,60.0 /signup,1,1,0,60.0
/some-other-page,1,1,,60.0 /some-other-page,1,1,0,60.0

1 name visitors pageviews bounce_rate time_on_page
2 / 4 3 67
3 /signup 1 1 0 60.0
4 /some-other-page 1 1 0 60.0

2
test/plausible_web/controllers/CSVs/6m/pages.csv
View File

@ -1,4 +1,4 @@
name,visitors,pageviews,bounce_rate,time_on_page name,visitors,pageviews,bounce_rate,time_on_page
/,5,4,75, /,5,4,75,
/signup,1,1,0,60.0 /signup,1,1,0,60.0
/some-other-page,1,1,,60.0 /some-other-page,1,1,0,60.0

1 name visitors pageviews bounce_rate time_on_page
2 / 5 4 75
3 /signup 1 1 0 60.0
4 /some-other-page 1 1 0 60.0

41
test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
View File

@ -890,8 +890,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
assert json_response(conn, 200) == %{ assert json_response(conn, 200) == %{
"results" => [ "results" => [
%{"page" => "/", "pageviews" => 2}, %{"page" => "/", "pageviews" => 2},
%{"page" => "/plausible.io", "pageviews" => 1}, %{"page" => "/include-me", "pageviews" => 1},
%{"page" => "/include-me", "pageviews" => 1} %{"page" => "/plausible.io", "pageviews" => 1}
] ]
} }
end end
@ -1023,7 +1023,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
"period" => "day", "period" => "day",
"date" => "2021-01-01", "date" => "2021-01-01",
"property" => "visit:exit_page", "property" => "visit:exit_page",
"metrics" => "visitors,visits,pageviews,bounce_rate,visit_duration,events", "metrics" => "visitors,visits,bounce_rate,visit_duration,events,pageviews",
"with_imported" => "true" "with_imported" => "true"
}) })
@ -1031,21 +1031,21 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
"results" => [ "results" => [
%{ %{
"bounce_rate" => 0.0, "bounce_rate" => 0.0,
"events" => 7,
"exit_page" => "/b", "exit_page" => "/b",
"pageviews" => 7,
"visit_duration" => 150.0, "visit_duration" => 150.0,
"visitors" => 3, "visitors" => 3,
"visits" => 4 "visits" => 4,
"events" => 7,
"pageviews" => 7
}, },
%{ %{
"bounce_rate" => 100.0, "bounce_rate" => 100.0,
"events" => 1,
"exit_page" => "/a", "exit_page" => "/a",
"pageviews" => 1,
"visit_duration" => 0.0, "visit_duration" => 0.0,
"visitors" => 1, "visitors" => 1,
"visits" => 1 "visits" => 1,
"events" => 1,
"pageviews" => 1
} }
] ]
} }
@ -2176,8 +2176,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
assert json_response(conn, 200) == %{ assert json_response(conn, 200) == %{
"results" => [ "results" => [
%{"page" => "/plausible.io", "bounce_rate" => 100}, %{"page" => "/important-page", "bounce_rate" => 100},
%{"page" => "/important-page", "bounce_rate" => 100} %{"page" => "/plausible.io", "bounce_rate" => 100}
] ]
} }
end end
@ -2596,14 +2596,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
assert json_response(conn, 200) == %{ assert json_response(conn, 200) == %{
"results" => [ "results" => [
%{
"page" => "/B",
"time_on_page" => 90.0
},
%{ %{
"page" => "/A", "page" => "/A",
"time_on_page" => 60.0 "time_on_page" => 60.0
}, },
%{
"page" => "/B",
"time_on_page" => 90.0
},
%{ %{
"page" => "/C", "page" => "/C",
"time_on_page" => nil "time_on_page" => nil
@ -3045,13 +3045,13 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
assert json_response(conn, 200) == %{ assert json_response(conn, 200) == %{
"results" => [ "results" => [
%{
"entry_page" => "/entry-page-1",
"bounce_rate" => 0
},
%{ %{
"entry_page" => "/entry-page-2", "entry_page" => "/entry-page-2",
"bounce_rate" => 100 "bounce_rate" => 100
},
%{
"entry_page" => "/entry-page-1",
"bounce_rate" => 0
} }
] ]
} }
@ -3146,6 +3146,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
}, },
%{ %{
"page" => "/plausible.io", "page" => "/plausible.io",
# Breaks for event:page breakdown since visitors is calculated based on entry_page :/
"visitors" => 2, "visitors" => 2,
"bounce_rate" => 100, "bounce_rate" => 100,
"visit_duration" => 0, "visit_duration" => 0,
@ -3290,8 +3291,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
assert %{"browser" => "Chrome", "events" => 1} = breakdown_and_first.("visit:browser") assert %{"browser" => "Chrome", "events" => 1} = breakdown_and_first.("visit:browser")
assert %{"device" => "Desktop", "events" => 1} = breakdown_and_first.("visit:device") assert %{"device" => "Desktop", "events" => 1} = breakdown_and_first.("visit:device")
assert %{"entry_page" => "/test", "events" => 1} = breakdown_and_first.("visit:entry_page")
assert %{"exit_page" => "/test", "events" => 1} = breakdown_and_first.("visit:exit_page")
assert %{"country" => "EE", "events" => 1} = breakdown_and_first.("visit:country") assert %{"country" => "EE", "events" => 1} = breakdown_and_first.("visit:country")
assert %{"os" => "Mac", "events" => 1} = breakdown_and_first.("visit:os") assert %{"os" => "Mac", "events" => 1} = breakdown_and_first.("visit:os")
assert %{"page" => "/test", "events" => 1} = breakdown_and_first.("event:page") assert %{"page" => "/test", "events" => 1} = breakdown_and_first.("event:page")

104
test/plausible_web/controllers/api/external_stats_controller/query_test.exs
View File

@ -1477,7 +1477,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
conn = conn =
post(conn, "/api/v2/query", %{ post(conn, "/api/v2/query", %{
"site_id" => site.domain, "site_id" => site.domain,
"metrics" => ["visitors"], "metrics" => ["visitors", "percentage"],
"date_range" => ["2021-01-01", "2021-01-01"], "date_range" => ["2021-01-01", "2021-01-01"],
"dimensions" => [unquote(dimension)] "dimensions" => [unquote(dimension)]
}) })
@ -1485,9 +1485,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
%{"results" => results} = json_response(conn, 200) %{"results" => results} = json_response(conn, 200)
assert results == [ assert results == [
%{"dimensions" => [unquote(value1)], "metrics" => [3]}, %{"dimensions" => [unquote(value1)], "metrics" => [3, 50]},
%{"dimensions" => [unquote(value2)], "metrics" => [2]}, %{"dimensions" => [unquote(value2)], "metrics" => [2, 33.3]},
%{"dimensions" => [unquote(blank_value)], "metrics" => [1]} %{"dimensions" => [unquote(blank_value)], "metrics" => [1, 16.7]}
] ]
end end
end end
@ -3463,6 +3463,48 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
%{"dimensions" => ["Chrome"], "metrics" => [1]} %{"dimensions" => ["Chrome"], "metrics" => [1]}
] ]
end end
test "all metrics for breakdown by event prop", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
user_id: 1,
pathname: "/",
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: 1,
pathname: "/plausible.io",
timestamp: ~N[2021-01-01 00:10:00]
),
build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview,
pathname: "/plausible.io",
timestamp: ~N[2021-01-01 00:00:00]
)
])
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => [
"visitors",
"visits",
"pageviews",
"events",
"bounce_rate",
"visit_duration"
],
"date_range" => "all",
"dimensions" => ["event:page"]
})
%{"results" => results} = json_response(conn, 200)
assert results == [
%{"dimensions" => ["/"], "metrics" => [2, 2, 2, 2, 50, 300]},
%{"dimensions" => ["/plausible.io"], "metrics" => [2, 2, 2, 2, 100, 0]}
]
end
end end
describe "imported data" do describe "imported data" do
@ -3589,10 +3631,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
assert %{"dimensions" => ["Desktop"], "metrics" => [1]} = assert %{"dimensions" => ["Desktop"], "metrics" => [1]} =
breakdown_and_first.("visit:device") breakdown_and_first.("visit:device")
# :TODO: These should not pass validation - not available on events.
# visit dimension and event-only metric
# assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("visit:entry_page")
# assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("visit:exit_page")
assert %{"dimensions" => ["EE"], "metrics" => [1]} = breakdown_and_first.("visit:country") assert %{"dimensions" => ["EE"], "metrics" => [1]} = breakdown_and_first.("visit:country")
assert %{"dimensions" => ["Mac"], "metrics" => [1]} = breakdown_and_first.("visit:os") assert %{"dimensions" => ["Mac"], "metrics" => [1]} = breakdown_and_first.("visit:os")
assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("event:page") assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("event:page")
@ -3865,6 +3903,56 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
assert meta["warning"] =~ "Imported stats are not included in the results" assert meta["warning"] =~ "Imported stats are not included in the results"
end end
test "imported country, region and city data",
%{
conn: conn,
site: site
} do
site_import = insert(:site_import, site: site)
populate_stats(site, site_import.id, [
build(:pageview,
timestamp: ~N[2021-01-01 00:15:00],
country_code: "DE",
subdivision1_code: "DE-BE",
city_geoname_id: 2_950_159
),
build(:pageview,
timestamp: ~N[2021-01-01 00:15:00],
country_code: "DE",
subdivision1_code: "DE-BE",
city_geoname_id: 2_950_159
),
build(:pageview,
timestamp: ~N[2021-01-01 00:15:00],
country_code: "EE",
subdivision1_code: "EE-37",
city_geoname_id: 588_409
),
build(:imported_locations, country: "EE", region: "EE-37", city: 588_409, visitors: 33)
])
for {dimension, stats_value, imports_value} <- [
{"visit:country", "DE", "EE"},
{"visit:region", "DE-BE", "EE-37"},
{"visit:city", 2_950_159, 588_409}
] do
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visitors"],
"date_range" => "all",
"dimensions" => [dimension],
"include" => %{"imports" => true}
})
assert json_response(conn, 200)["results"] == [
%{"dimensions" => [imports_value], "metrics" => [34]},
%{"dimensions" => [stats_value], "metrics" => [2]}
]
end
end
end end
test "multiple breakdown timeseries with sources", %{conn: conn, site: site} do test "multiple breakdown timeseries with sources", %{conn: conn, site: site} do

1
test/plausible_web/controllers/api/stats_controller/conversions_test.exs
View File

@ -207,6 +207,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
] ]
end end
@tag capture_log: true
test "garbage filters don't crash the call", %{conn: conn, site: site} do test "garbage filters don't crash the call", %{conn: conn, site: site} do
filters = filters =
"{\"source\":\"Direct / None\",\"screen\":\"Desktop\",\"browser\":\"Chrome\",\"os\":\"Mac\",\"os_version\":\"10.15\",\"country\":\"DE\",\"city\":\"2950159\"}%' AND 2*3*8=6*8 AND 'L9sv'!='L9sv%" "{\"source\":\"Direct / None\",\"screen\":\"Desktop\",\"browser\":\"Chrome\",\"os\":\"Mac\",\"os_version\":\"10.15\",\"country\":\"DE\",\"city\":\"2950159\"}%' AND 2*3*8=6*8 AND 'L9sv'!='L9sv%"

14
test/plausible_web/controllers/api/stats_controller/imported_test.exs
View File

@ -601,17 +601,17 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
) )
assert json_response(conn, 200)["results"] == [ assert json_response(conn, 200)["results"] == [
%{
"name" => "blog",
"visitors" => 2,
"bounce_rate" => 50.0,
"visit_duration" => 50.0
},
%{ %{
"name" => "ad", "name" => "ad",
"visitors" => 2, "visitors" => 2,
"bounce_rate" => 100.0, "bounce_rate" => 100.0,
"visit_duration" => 50.0 "visit_duration" => 50.0
},
%{
"name" => "blog",
"visitors" => 2,
"bounce_rate" => 50.0,
"visit_duration" => 50.0
} }
] ]
end end
@ -708,7 +708,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
assert json_response(conn, 200)["results"] == [ assert json_response(conn, 200)["results"] == [
%{ %{
"bounce_rate" => nil, "bounce_rate" => 0,
"time_on_page" => 60, "time_on_page" => 60,
"visitors" => 3, "visitors" => 3,
"pageviews" => 4, "pageviews" => 4,

16
test/plausible_web/controllers/api/stats_controller/pages_test.exs
View File

@ -340,7 +340,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"name" => "/blog/other-post", "name" => "/blog/other-post",
"visitors" => 1, "visitors" => 1,
"pageviews" => 1, "pageviews" => 1,
"bounce_rate" => nil, "bounce_rate" => 0,
"time_on_page" => nil "time_on_page" => nil
} }
] ]
@ -392,7 +392,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"name" => "/blog/other-post", "name" => "/blog/other-post",
"visitors" => 1, "visitors" => 1,
"pageviews" => 1, "pageviews" => 1,
"bounce_rate" => nil, "bounce_rate" => 0,
"time_on_page" => nil "time_on_page" => nil
} }
] ]
@ -744,7 +744,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"name" => "/blog/post-2", "name" => "/blog/post-2",
"visitors" => 1, "visitors" => 1,
"pageviews" => 1, "pageviews" => 1,
"bounce_rate" => nil, "bounce_rate" => 0,
"time_on_page" => nil "time_on_page" => nil
} }
] ]
@ -789,7 +789,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"name" => "/blog/(/post-2", "name" => "/blog/(/post-2",
"visitors" => 1, "visitors" => 1,
"pageviews" => 1, "pageviews" => 1,
"bounce_rate" => nil, "bounce_rate" => 0,
"time_on_page" => nil "time_on_page" => nil
} }
] ]
@ -842,7 +842,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"name" => "/about", "name" => "/about",
"visitors" => 1, "visitors" => 1,
"pageviews" => 1, "pageviews" => 1,
"bounce_rate" => nil, "bounce_rate" => 0,
"time_on_page" => nil "time_on_page" => nil
} }
] ]
@ -940,7 +940,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"name" => "/" "name" => "/"
}, },
%{ %{
"bounce_rate" => nil, "bounce_rate" => 0,
"time_on_page" => nil, "time_on_page" => nil,
"visitors" => 1, "visitors" => 1,
"pageviews" => 1, "pageviews" => 1,
@ -1066,7 +1066,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"visitors" => 2 "visitors" => 2
}, },
%{ %{
"bounce_rate" => nil, "bounce_rate" => 0,
"name" => "/exit-blog", "name" => "/exit-blog",
"pageviews" => 1, "pageviews" => 1,
"time_on_page" => nil, "time_on_page" => nil,
@ -1192,7 +1192,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"name" => "/" "name" => "/"
}, },
%{ %{
"bounce_rate" => nil, "bounce_rate" => 0,
"time_on_page" => 60, "time_on_page" => 60,
"visitors" => 2, "visitors" => 2,
"pageviews" => 2, "pageviews" => 2,

29
test/plausible_web/controllers/api/stats_controller/sources_test.exs
View File

@ -453,6 +453,9 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
referrer_source: "DuckDuckGo", referrer_source: "DuckDuckGo",
referrer: "duckduckgo.com" referrer: "duckduckgo.com"
), ),
build(:imported_sources,
source: "DuckDuckGo"
),
build(:imported_sources, build(:imported_sources,
source: "DuckDuckGo" source: "DuckDuckGo"
) )
@ -467,7 +470,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
conn = get(conn, "/api/stats/#{site.domain}/sources?limit=1&page=2&with_imported=true") conn = get(conn, "/api/stats/#{site.domain}/sources?limit=1&page=2&with_imported=true")
assert json_response(conn, 200)["results"] == [ assert json_response(conn, 200)["results"] == [
%{"name" => "DuckDuckGo", "visitors" => 2} %{"name" => "Google", "visitors" => 2}
] ]
end end
@ -590,17 +593,17 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
) )
assert json_response(conn, 200)["results"] == [ assert json_response(conn, 200)["results"] == [
%{
"name" => "social",
"visitors" => 1,
"bounce_rate" => 0,
"visit_duration" => 900
},
%{ %{
"name" => "email", "name" => "email",
"visitors" => 1, "visitors" => 1,
"bounce_rate" => 100, "bounce_rate" => 100,
"visit_duration" => 0 "visit_duration" => 0
},
%{
"name" => "social",
"visitors" => 1,
"bounce_rate" => 0,
"visit_duration" => 900
} }
] ]
@ -611,17 +614,17 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
) )
assert json_response(conn, 200)["results"] == [ assert json_response(conn, 200)["results"] == [
%{
"name" => "social",
"visitors" => 2,
"bounce_rate" => 50,
"visit_duration" => 800.0
},
%{ %{
"name" => "email", "name" => "email",
"visitors" => 2, "visitors" => 2,
"bounce_rate" => 50, "bounce_rate" => 50,
"visit_duration" => 50 "visit_duration" => 50
},
%{
"name" => "social",
"visitors" => 2,
"bounce_rate" => 50,
"visit_duration" => 800.0
} }
] ]
end end