mirror of
https://github.com/plausible/analytics.git
synced 2024-08-17 05:40:24 +03:00
APIv2: Replace breakdown module with QueryBuilder (#4283)
* WIP: Breakdown using QueryBuilder
* Revert "Remove problematic test"
This reverts commit b442bb5d1f
.
* Get more breakdown tests passing
* Preload goals, sort when dealing with time_on_page
* Handle conversion_rate in breakdowns
* Simplify ordering by using selected_as consistently for dimensions
* Get breakdown tests passing
* Strings to atoms in keys for StatsController.transform_keys calls to work
* Handle revenue metrics removal
* Add test for nil-removal case
* Include percentage metric
* Fix and test with imported locations
* Fixup time-on-page
* Fix country/region automatic filters
* Handle multiple imports (os/browser version) in importsv2
* Filter goals
* Default to ordering by page as well
* Calculate conversion rate on sessions if needed
* Order by event dimensions - handles event:page special case
* Update tests
* Update more tests, handle goal=0 case in imports
* Handle event:goal breakdowns correctly with filters
* Revenue to money
* Improved table deciding
* Also update event:page filters on event:page breakdown
* bounce_rate to 0
Previous behavior relied on two queries being made - new query leads to 0 naturally
* Update pagination test
* dont count non-pageviews as path goal completions
* Make revenue logic breakdown-specific
Its hard to fit into the new schema and likely needs a rethink for apiv2
* Retain previous behavior for TimeSeries module
* Get GA4 test passing
Most failures are related to ordering, pageviews shouldnt be read off of sessions
* Clean up old methods
* Simplify imported.ex
* Dont crash on garbage filters
* Reflect ordering-related change in test
* Fix test data
* Update table_decider
* Re-simplify get_revenue_tracking_currency
* Revert revenue changes
* Use Query.set
* Remove a TODO
* csv importer: no pageviews
Pageviews were incorrectly fetched from sessions table before, causing issues
* csv importer tweaking
* Remove use Plausible
* to_existing_atom
This commit is contained in:
parent
f4e091452c
commit
7dd12d1dd6
@ -16,6 +16,7 @@ All notable changes to this project will be documented in this file.
|
||||
- Increase hourly request limit for API keys in CE from 600 to 1000000 (practically removing the limit) plausible/analytics#4200
|
||||
- Make TCP connections try IPv6 first with IPv4 fallback in CE plausible/analytics#4245
|
||||
- `is` and `is not` filters in dashboard no longer support wildcards. Use contains/does not contain filter instead.
|
||||
- `bounce_rate` metric now returns 0 instead of null for event:page breakdown when page has never been entry page.
|
||||
|
||||
### Fixed
|
||||
|
||||
|
@ -250,6 +250,8 @@ defmodule Plausible.Stats.Base do
|
||||
end
|
||||
|
||||
defp select_session_metric(:percentage, _query), do: %{}
|
||||
defp select_session_metric(:conversion_rate, _query), do: %{}
|
||||
defp select_session_metric(:group_conversion_rate, _query), do: %{}
|
||||
|
||||
def filter_converted_sessions(db_query, site, query) do
|
||||
if Query.has_event_filters?(query) do
|
||||
|
@ -3,344 +3,61 @@ defmodule Plausible.Stats.Breakdown do
|
||||
use Plausible
|
||||
use Plausible.Stats.Fragments
|
||||
|
||||
import Plausible.Stats.{Base, Imported}
|
||||
import Plausible.Stats.Base
|
||||
import Ecto.Query
|
||||
alias Plausible.Stats.{Query, Util, TableDecider}
|
||||
alias Plausible.Stats.{Query, QueryOptimizer, QueryResult, SQL}
|
||||
alias Plausible.Stats.Filters.QueryParser
|
||||
|
||||
@no_ref "Direct / None"
|
||||
@not_set "(not set)"
|
||||
def breakdown(site, %Query{dimensions: [dimension]} = query, metrics, pagination, _opts \\ []) do
|
||||
transformed_metrics = transform_metrics(metrics, dimension)
|
||||
|
||||
@session_metrics [:bounce_rate, :visit_duration]
|
||||
|
||||
@revenue_metrics on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: [])
|
||||
|
||||
@event_metrics [:visits, :visitors, :pageviews, :events, :percentage] ++ @revenue_metrics
|
||||
|
||||
# These metrics can be asked from the `breakdown/5` function,
|
||||
# but they are different from regular metrics such as `visitors`,
|
||||
# or `bounce_rate` - we cannot currently "select them" directly in
|
||||
# the db queries. Instead, we need to artificially append them to
|
||||
# the breakdown results later on.
|
||||
@computed_metrics [:conversion_rate, :total_visitors]
|
||||
|
||||
def breakdown(site, query, metrics, pagination, opts \\ [])
|
||||
|
||||
def breakdown(
|
||||
site,
|
||||
%Query{dimensions: ["event:goal"]} = query,
|
||||
metrics,
|
||||
pagination,
|
||||
opts
|
||||
) do
|
||||
site = Plausible.Repo.preload(site, :goals)
|
||||
|
||||
{event_goals, pageview_goals} = Enum.split_with(site.goals, & &1.event_name)
|
||||
events = Enum.map(event_goals, & &1.event_name)
|
||||
|
||||
event_query =
|
||||
query
|
||||
|> Query.put_filter([:is, "event:name", events])
|
||||
|> Query.set_dimensions(["event:name"])
|
||||
|
||||
if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)
|
||||
|
||||
no_revenue = {nil, metrics -- @revenue_metrics}
|
||||
|
||||
{revenue_goals, metrics} =
|
||||
on_ee do
|
||||
if Plausible.Billing.Feature.RevenueGoals.enabled?(site) do
|
||||
revenue_goals = Enum.filter(event_goals, &Plausible.Goal.Revenue.revenue?/1)
|
||||
metrics = if Enum.empty?(revenue_goals), do: metrics -- @revenue_metrics, else: metrics
|
||||
|
||||
{revenue_goals, metrics}
|
||||
else
|
||||
no_revenue
|
||||
end
|
||||
else
|
||||
no_revenue
|
||||
end
|
||||
|
||||
metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics
|
||||
|
||||
event_q =
|
||||
if Enum.any?(event_goals) do
|
||||
site
|
||||
|> breakdown_events(event_query, metrics_to_select)
|
||||
|> apply_pagination(pagination)
|
||||
else
|
||||
nil
|
||||
end
|
||||
|
||||
page_q =
|
||||
if Enum.any?(pageview_goals) do
|
||||
page_query = Query.set_dimensions(query, ["event:page"])
|
||||
|
||||
page_exprs = Enum.map(pageview_goals, & &1.page_path)
|
||||
page_regexes = Enum.map(page_exprs, &page_regex/1)
|
||||
|
||||
select_columns = metrics_to_select |> select_event_metrics |> mark_revenue_as_nil
|
||||
|
||||
from(e in base_event_query(site, page_query),
|
||||
order_by: [desc: fragment("uniq(?)", e.user_id)],
|
||||
where:
|
||||
fragment(
|
||||
"notEmpty(multiMatchAllIndices(?, ?) as indices)",
|
||||
e.pathname,
|
||||
^page_regexes
|
||||
) and e.name == "pageview",
|
||||
array_join: index in fragment("indices"),
|
||||
group_by: index,
|
||||
select: %{
|
||||
name: fragment("concat('Visit ', ?[?])", ^page_exprs, index)
|
||||
}
|
||||
)
|
||||
|> select_merge(^select_columns)
|
||||
|> merge_imported_pageview_goals(site, page_query, page_exprs, metrics_to_select)
|
||||
|> apply_pagination(pagination)
|
||||
else
|
||||
nil
|
||||
end
|
||||
|
||||
full_q =
|
||||
case {event_q, page_q} do
|
||||
{nil, nil} ->
|
||||
nil
|
||||
|
||||
{event_q, nil} ->
|
||||
event_q
|
||||
|
||||
{nil, page_q} ->
|
||||
page_q
|
||||
|
||||
{event_q, page_q} ->
|
||||
from(
|
||||
e in subquery(union_all(event_q, ^page_q)),
|
||||
order_by: [desc: e.visitors]
|
||||
)
|
||||
|> apply_pagination(pagination)
|
||||
end
|
||||
|
||||
if full_q do
|
||||
full_q
|
||||
|> maybe_add_conversion_rate(site, query, metrics)
|
||||
|> ClickhouseRepo.all()
|
||||
|> transform_keys(%{name: :goal})
|
||||
|> cast_revenue_metrics_to_money(revenue_goals)
|
||||
|> Util.keep_requested_metrics(metrics)
|
||||
else
|
||||
[]
|
||||
end
|
||||
end
|
||||
|
||||
def breakdown(
|
||||
site,
|
||||
%Query{dimensions: ["event:props:" <> custom_prop]} = query,
|
||||
metrics,
|
||||
pagination,
|
||||
opts
|
||||
) do
|
||||
{currency, metrics} =
|
||||
on_ee do
|
||||
Plausible.Stats.Goal.Revenue.get_revenue_tracking_currency(site, query, metrics)
|
||||
else
|
||||
{nil, metrics}
|
||||
end
|
||||
|
||||
metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics
|
||||
|
||||
if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)
|
||||
|
||||
breakdown_events(site, query, metrics_to_select)
|
||||
|> maybe_add_conversion_rate(site, query, metrics)
|
||||
|> paginate_and_execute(metrics, pagination)
|
||||
|> transform_keys(%{breakdown_prop_value: custom_prop})
|
||||
|> Enum.map(&cast_revenue_metrics_to_money(&1, currency))
|
||||
end
|
||||
|
||||
def breakdown(site, %Query{dimensions: ["event:page"]} = query, metrics, pagination, opts) do
|
||||
event_metrics =
|
||||
metrics
|
||||
|> Util.maybe_add_visitors_metric()
|
||||
|> Enum.filter(&(&1 in @event_metrics))
|
||||
|
||||
if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)
|
||||
|
||||
event_result =
|
||||
site
|
||||
|> breakdown_events(query, event_metrics)
|
||||
|> maybe_add_group_conversion_rate(&breakdown_events/3, site, query, metrics)
|
||||
|> paginate_and_execute(metrics, pagination)
|
||||
|> maybe_add_time_on_page(site, query, metrics)
|
||||
|
||||
session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))
|
||||
|
||||
entry_page_query =
|
||||
case event_result do
|
||||
[] ->
|
||||
query
|
||||
|
||||
pages ->
|
||||
query
|
||||
|> Query.remove_filters(["event:page"])
|
||||
|> Query.put_filter([:is, "visit:entry_page", Enum.map(pages, & &1[:page])])
|
||||
|> Query.set_dimensions(["visit:entry_page"])
|
||||
end
|
||||
|
||||
if Enum.any?(event_metrics) && Enum.empty?(event_result) do
|
||||
[]
|
||||
else
|
||||
{limit, _page} = pagination
|
||||
|
||||
session_result =
|
||||
breakdown_sessions(site, entry_page_query, session_metrics)
|
||||
|> paginate_and_execute(session_metrics, {limit, 1})
|
||||
|> transform_keys(%{entry_page: :page})
|
||||
|
||||
metrics = metrics ++ [:page]
|
||||
|
||||
zip_results(
|
||||
event_result,
|
||||
session_result,
|
||||
:page,
|
||||
metrics
|
||||
query_with_metrics =
|
||||
Query.set(
|
||||
query,
|
||||
metrics: transformed_metrics,
|
||||
order_by: infer_order_by(transformed_metrics, dimension),
|
||||
dimensions: transform_dimensions(dimension),
|
||||
filters: query.filters ++ dimension_filters(dimension),
|
||||
preloaded_goals: QueryParser.preload_goals_if_needed(site, query.filters, [dimension]),
|
||||
v2: true
|
||||
)
|
||||
|> Enum.map(&Map.take(&1, metrics))
|
||||
end
|
||||
end
|
||||
|> QueryOptimizer.optimize()
|
||||
|
||||
def breakdown(site, %Query{dimensions: ["event:name"]} = query, metrics, pagination, opts) do
|
||||
if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)
|
||||
q = SQL.QueryBuilder.build(query_with_metrics, site)
|
||||
|
||||
breakdown_events(site, query, metrics)
|
||||
|> paginate_and_execute(metrics, pagination)
|
||||
end
|
||||
|
||||
def breakdown(site, query, metrics, pagination, opts) do
|
||||
query = maybe_update_breakdown_filters(query)
|
||||
if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)
|
||||
|
||||
metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics
|
||||
|
||||
case breakdown_table(query, metrics) do
|
||||
:session ->
|
||||
breakdown_sessions(site, query, metrics_to_select)
|
||||
|> maybe_add_group_conversion_rate(&breakdown_sessions/3, site, query, metrics)
|
||||
|> paginate_and_execute(metrics, pagination)
|
||||
|
||||
:event ->
|
||||
breakdown_events(site, query, metrics_to_select)
|
||||
|> maybe_add_group_conversion_rate(&breakdown_events/3, site, query, metrics)
|
||||
|> paginate_and_execute(metrics, pagination)
|
||||
end
|
||||
end
|
||||
|
||||
defp maybe_update_breakdown_filters(%Query{dimensions: [visit_entry_prop]} = query)
|
||||
when visit_entry_prop in [
|
||||
"visit:source",
|
||||
"visit:entry_page",
|
||||
"visit:utm_medium",
|
||||
"visit:utm_source",
|
||||
"visit:utm_campaign",
|
||||
"visit:utm_content",
|
||||
"visit:utm_term",
|
||||
"visit:entry_page",
|
||||
"visit:referrer"
|
||||
] do
|
||||
update_hostname_filter_prop(query, "visit:entry_page_hostname")
|
||||
end
|
||||
|
||||
defp maybe_update_breakdown_filters(%Query{dimensions: ["visit:exit_page"]} = query) do
|
||||
update_hostname_filter_prop(query, "visit:exit_page_hostname")
|
||||
end
|
||||
|
||||
defp maybe_update_breakdown_filters(query) do
|
||||
query
|
||||
end
|
||||
|
||||
defp update_hostname_filter_prop(query, visit_prop) do
|
||||
case Query.get_filter(query, "event:hostname") do
|
||||
nil ->
|
||||
query
|
||||
|
||||
[op, "event:hostname", value] ->
|
||||
query
|
||||
|> Query.put_filter([op, visit_prop, value])
|
||||
end
|
||||
end
|
||||
|
||||
# Backwards compatibility
|
||||
defp breakdown_table(%Query{experimental_reduced_joins?: false}, _), do: :session
|
||||
|
||||
defp breakdown_table(%Query{dimensions: ["visit:entry_page"]}, _metrics), do: :session
|
||||
defp breakdown_table(%Query{dimensions: ["visit:entry_page_hostname"]}, _metrics), do: :session
|
||||
defp breakdown_table(%Query{dimensions: ["visit:exit_page"]}, _metrics), do: :session
|
||||
defp breakdown_table(%Query{dimensions: ["visit:exit_page_hostname"]}, _metrics), do: :session
|
||||
|
||||
defp breakdown_table(%Query{dimensions: [_dimension]} = query, metrics) do
|
||||
{_, session_metrics, _} = TableDecider.partition_metrics(metrics, query)
|
||||
|
||||
if not Enum.empty?(session_metrics) do
|
||||
:session
|
||||
else
|
||||
:event
|
||||
end
|
||||
end
|
||||
|
||||
defp zip_results(event_result, session_result, property, metrics) do
|
||||
null_row = Enum.map(metrics, fn metric -> {metric, nil} end) |> Enum.into(%{})
|
||||
|
||||
prop_values =
|
||||
Enum.map(event_result ++ session_result, fn row -> row[property] end)
|
||||
|> Enum.uniq()
|
||||
|
||||
Enum.map(prop_values, fn value ->
|
||||
event_row = Enum.find(event_result, fn row -> row[property] == value end) || %{}
|
||||
session_row = Enum.find(session_result, fn row -> row[property] == value end) || %{}
|
||||
|
||||
null_row
|
||||
|> Map.merge(event_row)
|
||||
|> Map.merge(session_row)
|
||||
end)
|
||||
|> sort_results(metrics)
|
||||
end
|
||||
|
||||
defp breakdown_sessions(site, %Query{dimensions: [dimension]} = query, metrics) do
|
||||
from(s in query_sessions(site, query),
|
||||
order_by: [desc: fragment("uniq(?)", s.user_id)],
|
||||
select: ^select_session_metrics(metrics, query)
|
||||
)
|
||||
|> filter_converted_sessions(site, query)
|
||||
|> do_group_by(dimension)
|
||||
|> merge_imported(site, query, metrics)
|
||||
|> add_percentage_metric(site, query, metrics)
|
||||
end
|
||||
|
||||
defp breakdown_events(site, %Query{dimensions: [dimension]} = query, metrics) do
|
||||
from(e in base_event_query(site, query),
|
||||
order_by: [desc: fragment("uniq(?)", e.user_id)],
|
||||
select: %{}
|
||||
)
|
||||
|> do_group_by(dimension)
|
||||
|> select_merge(^select_event_metrics(metrics))
|
||||
|> merge_imported(site, query, metrics)
|
||||
|> add_percentage_metric(site, query, metrics)
|
||||
end
|
||||
|
||||
defp paginate_and_execute(_, [], _), do: []
|
||||
|
||||
defp paginate_and_execute(q, metrics, pagination) do
|
||||
q
|
||||
|> apply_pagination(pagination)
|
||||
|> ClickhouseRepo.all()
|
||||
|> Util.keep_requested_metrics(metrics)
|
||||
|> QueryResult.from(query_with_metrics)
|
||||
|> build_breakdown_result(query_with_metrics, metrics)
|
||||
|> maybe_add_time_on_page(site, query_with_metrics, metrics)
|
||||
|> update_currency_metrics(site, query_with_metrics)
|
||||
end
|
||||
|
||||
defp build_breakdown_result(query_result, query, metrics) do
|
||||
query_result.results
|
||||
|> Enum.map(fn %{dimensions: dimensions, metrics: entry_metrics} ->
|
||||
dimension_map =
|
||||
query.dimensions |> Enum.map(&result_key/1) |> Enum.zip(dimensions) |> Enum.into(%{})
|
||||
|
||||
metrics_map = Enum.zip(metrics, entry_metrics) |> Enum.into(%{})
|
||||
|
||||
Map.merge(dimension_map, metrics_map)
|
||||
end)
|
||||
end
|
||||
|
||||
defp result_key("event:props:" <> custom_property), do: custom_property
|
||||
defp result_key("event:" <> key), do: key |> String.to_existing_atom()
|
||||
defp result_key("visit:" <> key), do: key |> String.to_existing_atom()
|
||||
defp result_key(dimension), do: dimension
|
||||
|
||||
defp maybe_add_time_on_page(event_results, site, query, metrics) do
|
||||
if :time_on_page in metrics do
|
||||
if query.dimensions == ["event:page"] and :time_on_page in metrics do
|
||||
pages = Enum.map(event_results, & &1[:page])
|
||||
time_on_page_result = breakdown_time_on_page(site, query, pages)
|
||||
|
||||
Enum.map(event_results, fn row ->
|
||||
event_results
|
||||
|> Enum.map(fn row ->
|
||||
Map.put(row, :time_on_page, time_on_page_result[row[:page]])
|
||||
end)
|
||||
else
|
||||
@ -431,346 +148,60 @@ defmodule Plausible.Stats.Breakdown do
|
||||
|> Map.new()
|
||||
end
|
||||
|
||||
defp do_group_by(
|
||||
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,
|
||||
"event:props:" <> prop
|
||||
) do
|
||||
from(
|
||||
e in q,
|
||||
select_merge: %{
|
||||
breakdown_prop_value:
|
||||
selected_as(
|
||||
fragment(
|
||||
"if(not empty(?), ?, '(none)')",
|
||||
get_by_key(e, :meta, ^prop),
|
||||
get_by_key(e, :meta, ^prop)
|
||||
),
|
||||
:breakdown_prop_value
|
||||
)
|
||||
},
|
||||
group_by: selected_as(:breakdown_prop_value),
|
||||
order_by: {:asc, selected_as(:breakdown_prop_value)}
|
||||
)
|
||||
end
|
||||
defp transform_metrics(metrics, dimension) do
|
||||
metrics =
|
||||
if is_nil(metric_to_order_by(metrics)) do
|
||||
metrics ++ [:visitors]
|
||||
else
|
||||
metrics
|
||||
end
|
||||
|
||||
defp do_group_by(
|
||||
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,
|
||||
"event:name"
|
||||
) do
|
||||
from(
|
||||
e in q,
|
||||
group_by: e.name,
|
||||
select_merge: %{name: e.name},
|
||||
order_by: {:asc, e.name}
|
||||
)
|
||||
end
|
||||
|
||||
defp do_group_by(
|
||||
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,
|
||||
"event:page"
|
||||
) do
|
||||
from(
|
||||
e in q,
|
||||
group_by: e.pathname,
|
||||
select_merge: %{page: e.pathname},
|
||||
order_by: {:asc, e.pathname}
|
||||
)
|
||||
end
|
||||
|
||||
defp do_group_by(q, "visit:source") do
|
||||
from(
|
||||
s in q,
|
||||
group_by: s.source,
|
||||
select_merge: %{
|
||||
source: fragment("if(empty(?), ?, ?)", s.source, @no_ref, s.source)
|
||||
},
|
||||
order_by: {:asc, s.source}
|
||||
)
|
||||
end
|
||||
|
||||
defp do_group_by(q, "visit:country") do
|
||||
from(
|
||||
s in q,
|
||||
where: s.country != "\0\0" and s.country != "ZZ",
|
||||
group_by: s.country,
|
||||
select_merge: %{country: s.country},
|
||||
order_by: {:asc, s.country}
|
||||
)
|
||||
end
|
||||
|
||||
defp do_group_by(q, "visit:region") do
|
||||
from(
|
||||
s in q,
|
||||
where: s.region != "",
|
||||
group_by: s.region,
|
||||
select_merge: %{region: s.region},
|
||||
order_by: {:asc, s.region}
|
||||
)
|
||||
end
|
||||
|
||||
defp do_group_by(q, "visit:city") do
|
||||
from(
|
||||
s in q,
|
||||
where: s.city != 0,
|
||||
group_by: s.city,
|
||||
select_merge: %{city: s.city},
|
||||
order_by: {:asc, s.city}
|
||||
)
|
||||
end
|
||||
|
||||
defp do_group_by(q, "visit:entry_page") do
|
||||
from(
|
||||
s in q,
|
||||
# Sessions without pageviews don't get entry_page assigned, hence they should get ignored
|
||||
where: s.entry_page != "",
|
||||
group_by: s.entry_page,
|
||||
select_merge: %{entry_page: s.entry_page},
|
||||
order_by: {:asc, s.entry_page}
|
||||
)
|
||||
end
|
||||
|
||||
defp do_group_by(q, "visit:exit_page") do
|
||||
from(
|
||||
s in q,
|
||||
# Sessions without pageviews don't get entry_page assigned, hence they should get ignored
|
||||
where: s.entry_page != "",
|
||||
group_by: s.exit_page,
|
||||
select_merge: %{exit_page: s.exit_page},
|
||||
order_by: {:asc, s.exit_page}
|
||||
)
|
||||
end
|
||||
|
||||
defp do_group_by(q, "visit:referrer") do
|
||||
from(
|
||||
s in q,
|
||||
group_by: s.referrer,
|
||||
select_merge: %{
|
||||
referrer: fragment("if(empty(?), ?, ?)", s.referrer, @no_ref, s.referrer)
|
||||
},
|
||||
order_by: {:asc, s.referrer}
|
||||
)
|
||||
end
|
||||
|
||||
defp do_group_by(q, "visit:utm_medium") do
|
||||
from(
|
||||
s in q,
|
||||
where: fragment("not empty(?)", s.utm_medium),
|
||||
group_by: s.utm_medium,
|
||||
select_merge: %{
|
||||
utm_medium: s.utm_medium
|
||||
}
|
||||
)
|
||||
end
|
||||
|
||||
defp do_group_by(q, "visit:utm_source") do
|
||||
from(
|
||||
s in q,
|
||||
where: fragment("not empty(?)", s.utm_source),
|
||||
group_by: s.utm_source,
|
||||
select_merge: %{
|
||||
utm_source: s.utm_source
|
||||
}
|
||||
)
|
||||
end
|
||||
|
||||
defp do_group_by(q, "visit:utm_campaign") do
|
||||
from(
|
||||
s in q,
|
||||
where: fragment("not empty(?)", s.utm_campaign),
|
||||
group_by: s.utm_campaign,
|
||||
select_merge: %{
|
||||
utm_campaign: s.utm_campaign
|
||||
}
|
||||
)
|
||||
end
|
||||
|
||||
defp do_group_by(q, "visit:utm_content") do
|
||||
from(
|
||||
s in q,
|
||||
where: fragment("not empty(?)", s.utm_content),
|
||||
group_by: s.utm_content,
|
||||
select_merge: %{
|
||||
utm_content: s.utm_content
|
||||
}
|
||||
)
|
||||
end
|
||||
|
||||
defp do_group_by(q, "visit:utm_term") do
|
||||
from(
|
||||
s in q,
|
||||
where: fragment("not empty(?)", s.utm_term),
|
||||
group_by: s.utm_term,
|
||||
select_merge: %{
|
||||
utm_term: s.utm_term
|
||||
}
|
||||
)
|
||||
end
|
||||
|
||||
defp do_group_by(q, "visit:device") do
|
||||
from(
|
||||
s in q,
|
||||
group_by: s.device,
|
||||
select_merge: %{
|
||||
device: fragment("if(empty(?), ?, ?)", s.device, @not_set, s.device)
|
||||
},
|
||||
order_by: {:asc, s.device}
|
||||
)
|
||||
end
|
||||
|
||||
defp do_group_by(q, "visit:os") do
|
||||
from(
|
||||
s in q,
|
||||
group_by: s.os,
|
||||
select_merge: %{
|
||||
os: fragment("if(empty(?), ?, ?)", s.os, @not_set, s.os)
|
||||
},
|
||||
order_by: {:asc, s.os}
|
||||
)
|
||||
end
|
||||
|
||||
defp do_group_by(q, "visit:os_version") do
|
||||
from(
|
||||
s in q,
|
||||
group_by: [s.os, s.os_version],
|
||||
select_merge: %{
|
||||
os: fragment("if(empty(?), ?, ?)", s.os, @not_set, s.os),
|
||||
os_version:
|
||||
fragment(
|
||||
"if(empty(?), ?, ?)",
|
||||
s.os_version,
|
||||
@not_set,
|
||||
s.os_version
|
||||
)
|
||||
},
|
||||
order_by: {:asc, s.os_version}
|
||||
)
|
||||
end
|
||||
|
||||
defp do_group_by(q, "visit:browser") do
|
||||
from(
|
||||
s in q,
|
||||
group_by: s.browser,
|
||||
select_merge: %{
|
||||
browser: fragment("if(empty(?), ?, ?)", s.browser, @not_set, s.browser)
|
||||
},
|
||||
order_by: {:asc, s.browser}
|
||||
)
|
||||
end
|
||||
|
||||
defp do_group_by(q, "visit:browser_version") do
|
||||
from(
|
||||
s in q,
|
||||
group_by: [s.browser, s.browser_version],
|
||||
select_merge: %{
|
||||
browser: fragment("if(empty(?), ?, ?)", s.browser, @not_set, s.browser),
|
||||
browser_version:
|
||||
fragment("if(empty(?), ?, ?)", s.browser_version, @not_set, s.browser_version)
|
||||
},
|
||||
order_by: {:asc, s.browser_version}
|
||||
)
|
||||
end
|
||||
|
||||
defp group_by_field_names("event:props:" <> _prop), do: [:name]
|
||||
defp group_by_field_names("visit:os_version"), do: [:os, :os_version]
|
||||
defp group_by_field_names("visit:browser_version"), do: [:browser, :browser_version]
|
||||
|
||||
defp group_by_field_names(property), do: [Plausible.Stats.Filters.without_prefix(property)]
|
||||
|
||||
defp on_matches_group_by(fields) do
|
||||
Enum.reduce(fields, nil, &fields_equal/2)
|
||||
end
|
||||
|
||||
defp outer_order_by(fields) do
|
||||
Enum.map(fields, fn field_name -> {:asc, dynamic([q], field(q, ^field_name))} end)
|
||||
end
|
||||
|
||||
defp fields_equal(field_name, nil),
|
||||
do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name))
|
||||
|
||||
defp fields_equal(field_name, condition),
|
||||
do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name) and ^condition)
|
||||
|
||||
defp sort_results(results, metrics) do
|
||||
Enum.sort_by(
|
||||
results,
|
||||
fn entry ->
|
||||
case entry[sorting_key(metrics)] do
|
||||
nil -> 0
|
||||
n -> n
|
||||
end
|
||||
end,
|
||||
:desc
|
||||
)
|
||||
end
|
||||
|
||||
# This function injects a conversion_rate metric into
|
||||
# a breakdown query. It is calculated as X / Y, where:
|
||||
#
|
||||
# * X is the number of conversions for a breakdown
|
||||
# result (conversion = number of visitors who
|
||||
# completed the filtered goal with the filtered
|
||||
# custom properties).
|
||||
#
|
||||
# * Y is the number of all visitors for this breakdown
|
||||
# result without the `event:goal` and `event:props:*`
|
||||
# filters.
|
||||
defp maybe_add_group_conversion_rate(
|
||||
q,
|
||||
breakdown_fn,
|
||||
site,
|
||||
%Query{dimensions: [dimension]} = query,
|
||||
metrics
|
||||
) do
|
||||
if :conversion_rate in metrics do
|
||||
breakdown_total_visitors_query =
|
||||
query |> Query.remove_filters(["event:goal", "event:props"])
|
||||
|
||||
breakdown_total_visitors_q =
|
||||
breakdown_fn.(site, breakdown_total_visitors_query, [:visitors])
|
||||
|
||||
from(e in subquery(q),
|
||||
left_join: c in subquery(breakdown_total_visitors_q),
|
||||
on: ^on_matches_group_by(group_by_field_names(dimension)),
|
||||
select_merge: %{
|
||||
total_visitors: c.visitors,
|
||||
conversion_rate:
|
||||
fragment(
|
||||
"if(? > 0, round(? / ? * 100, 1), 0)",
|
||||
c.visitors,
|
||||
e.visitors,
|
||||
c.visitors
|
||||
)
|
||||
},
|
||||
order_by: [desc: e.visitors],
|
||||
order_by: ^outer_order_by(group_by_field_names(dimension))
|
||||
)
|
||||
else
|
||||
q
|
||||
end
|
||||
end
|
||||
|
||||
# When querying custom event goals and pageviewgoals together, UNION ALL is used
|
||||
# so the same fields must be present on both sides of the union. This change to the
|
||||
# query will ensure that we don't unnecessarily read revenue column for pageview goals
|
||||
defp mark_revenue_as_nil(select_columns) do
|
||||
select_columns
|
||||
|> Map.replace(:total_revenue, nil)
|
||||
|> Map.replace(:average_revenue, nil)
|
||||
end
|
||||
|
||||
defp sorting_key(metrics) do
|
||||
if Enum.member?(metrics, :visitors), do: :visitors, else: List.first(metrics)
|
||||
end
|
||||
|
||||
defp transform_keys(results, keys_to_replace) do
|
||||
Enum.map(results, fn map ->
|
||||
Enum.map(map, fn {key, val} ->
|
||||
{Map.get(keys_to_replace, key, key), val}
|
||||
end)
|
||||
|> Enum.into(%{})
|
||||
Enum.map(metrics, fn metric ->
|
||||
case {metric, dimension} do
|
||||
{:conversion_rate, "event:props:" <> _} -> :conversion_rate
|
||||
{:conversion_rate, "event:goal"} -> :conversion_rate
|
||||
{:conversion_rate, _} -> :group_conversion_rate
|
||||
_ -> metric
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp infer_order_by(metrics, "event:goal"), do: [{metric_to_order_by(metrics), :desc}]
|
||||
|
||||
defp infer_order_by(metrics, dimension),
|
||||
do: [{metric_to_order_by(metrics), :desc}, {dimension, :asc}]
|
||||
|
||||
defp metric_to_order_by(metrics) do
|
||||
Enum.find(metrics, &(&1 != :time_on_page))
|
||||
end
|
||||
|
||||
def transform_dimensions("visit:browser_version"),
|
||||
do: ["visit:browser", "visit:browser_version"]
|
||||
|
||||
def transform_dimensions("visit:os_version"), do: ["visit:os", "visit:os_version"]
|
||||
def transform_dimensions(dimension), do: [dimension]
|
||||
|
||||
@filter_dimensions_not %{
|
||||
"visit:city" => [0],
|
||||
"visit:country" => ["\0\0", "ZZ"],
|
||||
"visit:region" => [""],
|
||||
"visit:utm_medium" => [""],
|
||||
"visit:utm_source" => [""],
|
||||
"visit:utm_campaign" => [""],
|
||||
"visit:utm_content" => [""],
|
||||
"visit:utm_term" => [""],
|
||||
"visit:entry_page" => [""],
|
||||
"visit:exit_page" => [""]
|
||||
}
|
||||
|
||||
@extra_filter_dimensions Map.keys(@filter_dimensions_not)
|
||||
|
||||
defp dimension_filters(dimension) when dimension in @extra_filter_dimensions do
|
||||
[[:is_not, dimension, Map.get(@filter_dimensions_not, dimension)]]
|
||||
end
|
||||
|
||||
defp dimension_filters(_), do: []
|
||||
|
||||
defp apply_pagination(q, {limit, page}) do
|
||||
offset = (page - 1) * limit
|
||||
|
||||
@ -780,10 +211,39 @@ defmodule Plausible.Stats.Breakdown do
|
||||
end
|
||||
|
||||
on_ee do
|
||||
defp cast_revenue_metrics_to_money(results, revenue_goals) do
|
||||
Plausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(results, revenue_goals)
|
||||
defp update_currency_metrics(results, site, %Query{dimensions: ["event:goal"]}) do
|
||||
site = Plausible.Repo.preload(site, :goals)
|
||||
|
||||
{event_goals, _pageview_goals} = Enum.split_with(site.goals, & &1.event_name)
|
||||
revenue_goals = Enum.filter(event_goals, &Plausible.Goal.Revenue.revenue?/1)
|
||||
|
||||
if length(revenue_goals) > 0 and Plausible.Billing.Feature.RevenueGoals.enabled?(site) do
|
||||
Plausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(results, revenue_goals)
|
||||
else
|
||||
remove_revenue_metrics(results)
|
||||
end
|
||||
end
|
||||
|
||||
defp update_currency_metrics(results, site, query) do
|
||||
{currency, _metrics} =
|
||||
Plausible.Stats.Goal.Revenue.get_revenue_tracking_currency(site, query, query.metrics)
|
||||
|
||||
if currency do
|
||||
results
|
||||
|> Enum.map(&Plausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(&1, currency))
|
||||
else
|
||||
remove_revenue_metrics(results)
|
||||
end
|
||||
end
|
||||
else
|
||||
defp cast_revenue_metrics_to_money(results, _revenue_goals), do: results
|
||||
defp update_currency_metrics(results, _site, _query), do: remove_revenue_metrics(results)
|
||||
end
|
||||
|
||||
defp remove_revenue_metrics(results) do
|
||||
Enum.map(results, fn map ->
|
||||
map
|
||||
|> Map.delete(:total_revenue)
|
||||
|> Map.delete(:average_revenue)
|
||||
end)
|
||||
end
|
||||
end
|
||||
|
@ -4,6 +4,7 @@ defmodule Plausible.Stats.Filters.QueryParser do
|
||||
alias Plausible.Stats.TableDecider
|
||||
alias Plausible.Stats.Filters
|
||||
alias Plausible.Stats.Query
|
||||
alias Plausible.Stats.Metrics
|
||||
|
||||
def parse(site, params, now \\ nil) when is_map(params) do
|
||||
with {:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])),
|
||||
@ -44,17 +45,12 @@ defmodule Plausible.Stats.Filters.QueryParser do
|
||||
|
||||
defp parse_metrics(_invalid_metrics), do: {:error, "Invalid metrics passed"}
|
||||
|
||||
defp parse_metric("time_on_page"), do: {:ok, :time_on_page}
|
||||
defp parse_metric("conversion_rate"), do: {:ok, :conversion_rate}
|
||||
defp parse_metric("group_conversion_rate"), do: {:ok, :group_conversion_rate}
|
||||
defp parse_metric("visitors"), do: {:ok, :visitors}
|
||||
defp parse_metric("pageviews"), do: {:ok, :pageviews}
|
||||
defp parse_metric("events"), do: {:ok, :events}
|
||||
defp parse_metric("visits"), do: {:ok, :visits}
|
||||
defp parse_metric("bounce_rate"), do: {:ok, :bounce_rate}
|
||||
defp parse_metric("visit_duration"), do: {:ok, :visit_duration}
|
||||
defp parse_metric("views_per_visit"), do: {:ok, :views_per_visit}
|
||||
defp parse_metric(unknown_metric), do: {:error, "Unknown metric '#{inspect(unknown_metric)}'"}
|
||||
defp parse_metric(metric_str) do
|
||||
case Metrics.from_string(metric_str) do
|
||||
{:ok, metric} -> {:ok, metric}
|
||||
_ -> {:error, "Unknown metric '#{inspect(metric_str)}'"}
|
||||
end
|
||||
end
|
||||
|
||||
def parse_filters(filters) when is_list(filters) do
|
||||
parse_list(filters, &parse_filter/1)
|
||||
@ -284,7 +280,7 @@ defmodule Plausible.Stats.Filters.QueryParser do
|
||||
end
|
||||
end
|
||||
|
||||
defp preload_goals_if_needed(site, filters, dimensions) do
|
||||
def preload_goals_if_needed(site, filters, dimensions) do
|
||||
goal_filters? =
|
||||
Enum.any?(filters, fn [_, filter_key | _rest] -> filter_key == "event:goal" end)
|
||||
|
||||
|
@ -10,6 +10,8 @@ defmodule Plausible.Stats.Filters.WhereBuilder do
|
||||
|
||||
use Plausible.Stats.Fragments
|
||||
|
||||
require Logger
|
||||
|
||||
@sessions_only_visit_fields [
|
||||
:entry_page,
|
||||
:exit_page,
|
||||
@ -150,6 +152,15 @@ defmodule Plausible.Stats.Filters.WhereBuilder do
|
||||
true
|
||||
end
|
||||
|
||||
defp add_filter(table, _query, filter) do
|
||||
Logger.info("Unable to process garbage filter. No results are returned",
|
||||
table: table,
|
||||
filter: filter
|
||||
)
|
||||
|
||||
false
|
||||
end
|
||||
|
||||
defp filter_custom_prop(prop_name, column_name, [:is, _, values]) do
|
||||
none_value_included = Enum.member?(values, "(none)")
|
||||
|
||||
|
@ -54,10 +54,12 @@ defmodule Plausible.Stats.Imported.Base do
|
||||
def property_to_table_mappings(), do: @property_to_table_mappings
|
||||
|
||||
def query_imported(site, query) do
|
||||
query
|
||||
|> transform_filters()
|
||||
|> decide_table()
|
||||
|> query_imported(site, query)
|
||||
[table] =
|
||||
query
|
||||
|> transform_filters()
|
||||
|> decide_tables()
|
||||
|
||||
query_imported(table, site, query)
|
||||
end
|
||||
|
||||
def query_imported(table, site, query) do
|
||||
@ -75,13 +77,13 @@ defmodule Plausible.Stats.Imported.Base do
|
||||
|> apply_filter(query)
|
||||
end
|
||||
|
||||
def decide_table(query) do
|
||||
def decide_tables(query) do
|
||||
query = transform_filters(query)
|
||||
|
||||
if custom_prop_query?(query) do
|
||||
do_decide_custom_prop_table(query)
|
||||
else
|
||||
do_decide_table(query)
|
||||
do_decide_tables(query)
|
||||
end
|
||||
end
|
||||
|
||||
@ -92,19 +94,17 @@ defmodule Plausible.Stats.Imported.Base do
|
||||
[:is, "event:name", ["pageview"]] -> true
|
||||
_ -> false
|
||||
end)
|
||||
|> Enum.flat_map(fn filter ->
|
||||
case filter do
|
||||
[op, "event:goal", events] ->
|
||||
events
|
||||
|> Enum.group_by(&elem(&1, 0), &elem(&1, 1))
|
||||
|> Enum.map(fn
|
||||
{:event, names} -> [op, "event:name", names]
|
||||
{:page, pages} -> [op, "event:page", pages]
|
||||
end)
|
||||
|> Enum.flat_map(fn
|
||||
[op, "event:goal", clauses] ->
|
||||
clauses
|
||||
|> Enum.group_by(&elem(&1, 0), &elem(&1, 1))
|
||||
|> Enum.map(fn
|
||||
{:event, names} -> [op, "event:name", names]
|
||||
{:page, pages} -> [op, "event:page", pages]
|
||||
end)
|
||||
|
||||
filter ->
|
||||
[filter]
|
||||
end
|
||||
filter ->
|
||||
[filter]
|
||||
end)
|
||||
|
||||
struct!(query, filters: new_filters)
|
||||
@ -136,10 +136,10 @@ defmodule Plausible.Stats.Imported.Base do
|
||||
do_decide_custom_prop_table(query, custom_prop_filter)
|
||||
|
||||
_ ->
|
||||
nil
|
||||
[]
|
||||
end
|
||||
else
|
||||
nil
|
||||
[]
|
||||
end
|
||||
end
|
||||
|
||||
@ -158,23 +158,27 @@ defmodule Plausible.Stats.Imported.Base do
|
||||
end)
|
||||
|
||||
if has_required_name_filter? and not has_unsupported_filters? do
|
||||
"imported_custom_events"
|
||||
["imported_custom_events"]
|
||||
else
|
||||
nil
|
||||
[]
|
||||
end
|
||||
end
|
||||
|
||||
defp do_decide_table(%Query{filters: [], dimensions: []}), do: "imported_visitors"
|
||||
defp do_decide_tables(%Query{filters: [], dimensions: []}), do: ["imported_visitors"]
|
||||
|
||||
defp do_decide_table(%Query{filters: [], dimensions: ["event:goal"]}) do
|
||||
"imported_custom_events"
|
||||
defp do_decide_tables(%Query{filters: [], dimensions: ["event:goal"]}) do
|
||||
["imported_pages", "imported_custom_events"]
|
||||
end
|
||||
|
||||
defp do_decide_table(%Query{filters: [], dimensions: [dimension]}) do
|
||||
@property_to_table_mappings[dimension]
|
||||
defp do_decide_tables(%Query{filters: [], dimensions: [dimension]}) do
|
||||
if Map.has_key?(@property_to_table_mappings, dimension) do
|
||||
[@property_to_table_mappings[dimension]]
|
||||
else
|
||||
[]
|
||||
end
|
||||
end
|
||||
|
||||
defp do_decide_table(%Query{filters: filters, dimensions: ["event:goal"]}) do
|
||||
defp do_decide_tables(%Query{filters: filters, dimensions: ["event:goal"]}) do
|
||||
filter_props = Enum.map(filters, &Enum.at(&1, 1))
|
||||
|
||||
any_event_name_filters? = "event:name" in filter_props
|
||||
@ -182,27 +186,28 @@ defmodule Plausible.Stats.Imported.Base do
|
||||
any_other_filters? = Enum.any?(filter_props, &(&1 not in ["event:page", "event:name"]))
|
||||
|
||||
cond do
|
||||
any_other_filters? -> nil
|
||||
any_event_name_filters? and not any_page_filters? -> "imported_custom_events"
|
||||
any_page_filters? and not any_event_name_filters? -> "imported_pages"
|
||||
true -> nil
|
||||
any_other_filters? -> []
|
||||
any_event_name_filters? and not any_page_filters? -> ["imported_custom_events"]
|
||||
any_page_filters? and not any_event_name_filters? -> ["imported_pages"]
|
||||
true -> []
|
||||
end
|
||||
end
|
||||
|
||||
defp do_decide_table(%Query{filters: filters, dimensions: dimensions}) do
|
||||
defp do_decide_tables(%Query{filters: filters, dimensions: dimensions}) do
|
||||
table_candidates =
|
||||
filters
|
||||
|> Enum.map(fn [_, filter_key | _] -> filter_key end)
|
||||
|> Enum.concat(dimensions)
|
||||
|> Enum.map(fn
|
||||
"visit:screen" -> "visit:device"
|
||||
prop -> prop
|
||||
dimension -> dimension
|
||||
end)
|
||||
|> Enum.map(&@property_to_table_mappings[&1])
|
||||
|
||||
case Enum.uniq(table_candidates) do
|
||||
[candidate] -> candidate
|
||||
_ -> nil
|
||||
[nil] -> []
|
||||
[candidate] -> [candidate]
|
||||
_ -> []
|
||||
end
|
||||
end
|
||||
|
||||
|
@ -4,10 +4,11 @@ defmodule Plausible.Stats.Imported do
|
||||
|
||||
import Ecto.Query
|
||||
import Plausible.Stats.Fragments
|
||||
import Plausible.Stats.Util, only: [shortname: 2]
|
||||
|
||||
alias Plausible.Stats.Base
|
||||
alias Plausible.Stats.Imported
|
||||
alias Plausible.Stats.Query
|
||||
alias Plausible.Stats.SQL.QueryBuilder
|
||||
|
||||
@no_ref "Direct / None"
|
||||
@not_set "(not set)"
|
||||
@ -15,7 +16,7 @@ defmodule Plausible.Stats.Imported do
|
||||
|
||||
@property_to_table_mappings Imported.Base.property_to_table_mappings()
|
||||
|
||||
@imported_properties Map.keys(@property_to_table_mappings) ++
|
||||
@imported_dimensions Map.keys(@property_to_table_mappings) ++
|
||||
Plausible.Imported.imported_custom_props()
|
||||
|
||||
@goals_with_url Plausible.Imported.goals_with_url()
|
||||
@ -37,7 +38,7 @@ defmodule Plausible.Stats.Imported do
|
||||
(see `@goals_with_url` and `@goals_with_path`).
|
||||
"""
|
||||
def schema_supports_query?(query) do
|
||||
not is_nil(Imported.Base.decide_table(query))
|
||||
length(Imported.Base.decide_tables(query)) > 0
|
||||
end
|
||||
|
||||
def merge_imported_country_suggestions(native_q, _site, %Plausible.Stats.Query{
|
||||
@ -267,78 +268,6 @@ defmodule Plausible.Stats.Imported do
|
||||
|
||||
def merge_imported(q, _, %Query{include_imported: false}, _), do: q
|
||||
|
||||
# Note: Only called for APIv2, old APIs use merge_imported_pageview_goals
|
||||
def merge_imported(q, site, %Query{dimensions: ["event:goal"]} = query, metrics)
|
||||
when query.v2 do
|
||||
{events, page_regexes} = Filters.Utils.split_goals_query_expressions(query.preloaded_goals)
|
||||
|
||||
events_q =
|
||||
"imported_custom_events"
|
||||
|> Imported.Base.query_imported(site, query)
|
||||
|> where([i], i.visitors > 0)
|
||||
|> select_merge([i], %{
|
||||
dim0: selected_as(fragment("-indexOf(?, ?)", ^events, i.name), :dim0)
|
||||
})
|
||||
|> select_imported_metrics(metrics)
|
||||
|> group_by([], selected_as(:dim0))
|
||||
|> where([], selected_as(:dim0) != 0)
|
||||
|
||||
pages_q =
|
||||
"imported_pages"
|
||||
|> Imported.Base.query_imported(site, query)
|
||||
|> where([i], i.visitors > 0)
|
||||
|> where(
|
||||
[i],
|
||||
fragment("notEmpty(multiMatchAllIndices(?, ?) as indices)", i.page, ^page_regexes)
|
||||
)
|
||||
|> join(:array, index in fragment("indices"))
|
||||
|> group_by([_i, index], index)
|
||||
|> select_merge([_i, index], %{
|
||||
dim0: type(fragment("?", index), :integer)
|
||||
})
|
||||
|> select_imported_metrics(metrics)
|
||||
|
||||
q
|
||||
|> naive_dimension_join(events_q, metrics)
|
||||
|> naive_dimension_join(pages_q, metrics)
|
||||
end
|
||||
|
||||
def merge_imported(q, site, %Query{dimensions: [dimension]} = query, metrics)
|
||||
when dimension in @imported_properties do
|
||||
dim = Plausible.Stats.Filters.without_prefix(dimension)
|
||||
|
||||
imported_q =
|
||||
site
|
||||
|> Imported.Base.query_imported(query)
|
||||
|> where([i], i.visitors > 0)
|
||||
|> group_imported_by(dim, query)
|
||||
|> select_imported_metrics(metrics)
|
||||
|
||||
join_on =
|
||||
case dim do
|
||||
_ when dim in [:url, :path] and not query.v2 ->
|
||||
dynamic([s, i], s.breakdown_prop_value == i.breakdown_prop_value)
|
||||
|
||||
:os_version when not query.v2 ->
|
||||
dynamic([s, i], s.os == i.os and s.os_version == i.os_version)
|
||||
|
||||
:browser_version when not query.v2 ->
|
||||
dynamic([s, i], s.browser == i.browser and s.browser_version == i.browser_version)
|
||||
|
||||
dim ->
|
||||
dynamic([s, i], field(s, ^shortname(query, dim)) == field(i, ^shortname(query, dim)))
|
||||
end
|
||||
|
||||
from(s in Ecto.Query.subquery(q),
|
||||
full_join: i in subquery(imported_q),
|
||||
on: ^join_on,
|
||||
select: %{}
|
||||
)
|
||||
|> select_joined_dimension(dim, query)
|
||||
|> select_joined_metrics(metrics)
|
||||
|> apply_order_by(query, metrics)
|
||||
end
|
||||
|
||||
def merge_imported(q, site, %Query{dimensions: []} = query, metrics) do
|
||||
imported_q =
|
||||
site
|
||||
@ -353,51 +282,82 @@ defmodule Plausible.Stats.Imported do
|
||||
|> select_joined_metrics(metrics)
|
||||
end
|
||||
|
||||
def merge_imported(q, _, _, _), do: q
|
||||
def merge_imported(q, site, %Query{dimensions: ["event:goal"]} = query, metrics) do
|
||||
{events, page_regexes} = Filters.Utils.split_goals_query_expressions(query.preloaded_goals)
|
||||
|
||||
def merge_imported_pageview_goals(q, _, %Query{include_imported: false}, _, _), do: q
|
||||
Imported.Base.decide_tables(query)
|
||||
|> Enum.map(fn
|
||||
"imported_custom_events" ->
|
||||
Imported.Base.query_imported("imported_custom_events", site, query)
|
||||
|> where([i], i.visitors > 0)
|
||||
|> select_merge([i], %{
|
||||
dim0:
|
||||
selected_as(
|
||||
fragment("-indexOf(?, ?)", type(^events, {:array, :string}), i.name),
|
||||
:dim0
|
||||
)
|
||||
})
|
||||
|> select_imported_metrics(metrics)
|
||||
|> group_by([], selected_as(:dim0))
|
||||
|> where([], selected_as(:dim0) != 0)
|
||||
|
||||
def merge_imported_pageview_goals(q, site, query, page_exprs, metrics) do
|
||||
if Imported.Base.decide_table(query) == "imported_pages" do
|
||||
page_regexes = Enum.map(page_exprs, &Base.page_regex/1)
|
||||
|
||||
imported_q =
|
||||
"imported_pages"
|
||||
|> Imported.Base.query_imported(site, query)
|
||||
"imported_pages" ->
|
||||
Imported.Base.query_imported("imported_pages", site, query)
|
||||
|> where([i], i.visitors > 0)
|
||||
|> where(
|
||||
[i],
|
||||
fragment("notEmpty(multiMatchAllIndices(?, ?) as indices)", i.page, ^page_regexes)
|
||||
fragment(
|
||||
"notEmpty(multiMatchAllIndices(?, ?) as indices)",
|
||||
i.page,
|
||||
type(^page_regexes, {:array, :string})
|
||||
)
|
||||
)
|
||||
|> join(:array, index in fragment("indices"))
|
||||
|> group_by([_i, index], index)
|
||||
|> select_merge([_i, index], %{
|
||||
name: fragment("concat('Visit ', ?[?])", ^page_exprs, index)
|
||||
dim0: selected_as(type(fragment("?", index), :integer), :dim0)
|
||||
})
|
||||
|> select_imported_metrics(metrics)
|
||||
end)
|
||||
|> Enum.reduce(q, fn imports_q, q ->
|
||||
naive_dimension_join(q, imports_q, metrics)
|
||||
end)
|
||||
end
|
||||
|
||||
from(s in Ecto.Query.subquery(q),
|
||||
def merge_imported(q, site, %Query{dimensions: dimensions} = query, metrics) do
|
||||
if merge_imported_dimensions?(dimensions) do
|
||||
imported_q =
|
||||
site
|
||||
|> Imported.Base.query_imported(query)
|
||||
|> where([i], i.visitors > 0)
|
||||
|> group_imported_by(query)
|
||||
|> select_imported_metrics(metrics)
|
||||
|
||||
from(s in subquery(q),
|
||||
full_join: i in subquery(imported_q),
|
||||
on: s.name == i.name,
|
||||
on: ^QueryBuilder.build_group_by_join(query),
|
||||
select: %{}
|
||||
)
|
||||
|> select_joined_dimension(:name, query)
|
||||
|> select_joined_dimensions(query)
|
||||
|> select_joined_metrics(metrics)
|
||||
else
|
||||
q
|
||||
end
|
||||
end
|
||||
|
||||
def merge_imported(q, _, _, _), do: q
|
||||
|
||||
defp merge_imported_dimensions?(dimensions) do
|
||||
dimensions in [["visit:browser", "visit:browser_version"], ["visit:os", "visit:os_version"]] or
|
||||
(length(dimensions) == 1 and hd(dimensions) in @imported_dimensions)
|
||||
end
|
||||
|
||||
def total_imported_visitors(site, query) do
|
||||
site
|
||||
|> Imported.Base.query_imported(query)
|
||||
|> select_merge([i], %{total_visitors: fragment("sum(?)", i.visitors)})
|
||||
end
|
||||
|
||||
# :TRICKY: Handle backwards compatibility with old breakdown module
|
||||
defp shortname(query, _dim) when query.v2, do: :dim0
|
||||
defp shortname(_query, dim), do: dim
|
||||
|
||||
defp select_imported_metrics(q, []), do: q
|
||||
|
||||
defp select_imported_metrics(q, [:visitors | rest]) do
|
||||
@ -592,190 +552,175 @@ defmodule Plausible.Stats.Imported do
|
||||
|> select_imported_metrics(rest)
|
||||
end
|
||||
|
||||
defp group_imported_by(q, dim, query) when dim in [:source, :referrer] do
|
||||
defp group_imported_by(q, query) do
|
||||
Enum.reduce(query.dimensions, q, fn dimension, q ->
|
||||
dim = Plausible.Stats.Filters.without_prefix(dimension)
|
||||
|
||||
group_imported_by(q, dim, shortname(query, dimension))
|
||||
end)
|
||||
end
|
||||
|
||||
defp group_imported_by(q, dim, key) when dim in [:source, :referrer] do
|
||||
q
|
||||
|> group_by([i], field(i, ^dim))
|
||||
|> select_merge([i], %{
|
||||
^shortname(query, dim) =>
|
||||
fragment(
|
||||
"if(empty(?), ?, ?)",
|
||||
field(i, ^dim),
|
||||
@no_ref,
|
||||
field(i, ^dim)
|
||||
^key =>
|
||||
selected_as(
|
||||
fragment(
|
||||
"if(empty(?), ?, ?)",
|
||||
field(i, ^dim),
|
||||
@no_ref,
|
||||
field(i, ^dim)
|
||||
),
|
||||
^key
|
||||
)
|
||||
})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, dim, query)
|
||||
defp group_imported_by(q, dim, key)
|
||||
when dim in [:utm_source, :utm_medium, :utm_campaign, :utm_term, :utm_content] do
|
||||
q
|
||||
|> group_by([i], field(i, ^dim))
|
||||
|> where([i], fragment("not empty(?)", field(i, ^dim)))
|
||||
|> select_merge([i], %{^shortname(query, dim) => field(i, ^dim)})
|
||||
|> select_merge([i], %{^key => selected_as(field(i, ^dim), ^key)})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, :page, query) do
|
||||
defp group_imported_by(q, :page, key) do
|
||||
q
|
||||
|> group_by([i], i.page)
|
||||
|> select_merge([i], %{^shortname(query, :page) => i.page, time_on_page: sum(i.time_on_page)})
|
||||
|> select_merge([i], %{^key => selected_as(i.page, ^key), time_on_page: sum(i.time_on_page)})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, :country, query) do
|
||||
defp group_imported_by(q, :country, key) do
|
||||
q
|
||||
|> group_by([i], i.country)
|
||||
|> where([i], i.country != "ZZ")
|
||||
|> select_merge([i], %{^shortname(query, :country) => i.country})
|
||||
|> select_merge([i], %{^key => selected_as(i.country, ^key)})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, :region, query) do
|
||||
defp group_imported_by(q, :region, key) do
|
||||
q
|
||||
|> group_by([i], i.region)
|
||||
|> where([i], i.region != "")
|
||||
|> select_merge([i], %{^shortname(query, :region) => i.region})
|
||||
|> select_merge([i], %{^key => selected_as(i.region, ^key)})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, :city, query) do
|
||||
defp group_imported_by(q, :city, key) do
|
||||
q
|
||||
|> group_by([i], i.city)
|
||||
|> where([i], i.city != 0 and not is_nil(i.city))
|
||||
|> select_merge([i], %{^shortname(query, :city) => i.city})
|
||||
|> select_merge([i], %{^key => selected_as(i.city, ^key)})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, dim, query) when dim in [:device, :browser] do
|
||||
defp group_imported_by(q, dim, key) when dim in [:device, :browser] do
|
||||
q
|
||||
|> group_by([i], field(i, ^dim))
|
||||
|> select_merge([i], %{
|
||||
^shortname(query, dim) =>
|
||||
fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim))
|
||||
})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, :browser_version, query) do
|
||||
q
|
||||
|> group_by([i], [i.browser, i.browser_version])
|
||||
|> select_merge([i], %{
|
||||
^shortname(query, :browser) =>
|
||||
fragment("if(empty(?), ?, ?)", i.browser, @not_set, i.browser),
|
||||
^shortname(query, :browser_version) =>
|
||||
fragment(
|
||||
"if(empty(?), ?, ?)",
|
||||
i.browser_version,
|
||||
@not_set,
|
||||
i.browser_version
|
||||
^key =>
|
||||
selected_as(
|
||||
fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim)),
|
||||
^key
|
||||
)
|
||||
})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, :os, query) do
|
||||
defp group_imported_by(q, :browser_version, key) do
|
||||
q
|
||||
|> group_by([i], [i.browser_version])
|
||||
|> select_merge([i], %{
|
||||
^key =>
|
||||
selected_as(
|
||||
fragment(
|
||||
"if(empty(?), ?, ?)",
|
||||
i.browser_version,
|
||||
@not_set,
|
||||
i.browser_version
|
||||
),
|
||||
^key
|
||||
)
|
||||
})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, :os, key) do
|
||||
q
|
||||
|> group_by([i], i.operating_system)
|
||||
|> select_merge([i], %{
|
||||
^shortname(query, :os) =>
|
||||
fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system)
|
||||
})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, :os_version, query) do
|
||||
q
|
||||
|> group_by([i], [i.operating_system, i.operating_system_version])
|
||||
|> select_merge([i], %{
|
||||
^shortname(query, :os) =>
|
||||
fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system),
|
||||
^shortname(query, :os_version) =>
|
||||
fragment(
|
||||
"if(empty(?), ?, ?)",
|
||||
i.operating_system_version,
|
||||
@not_set,
|
||||
i.operating_system_version
|
||||
^key =>
|
||||
selected_as(
|
||||
fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system),
|
||||
^key
|
||||
)
|
||||
})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, dim, query) when dim in [:entry_page, :exit_page] do
|
||||
defp group_imported_by(q, :os_version, key) do
|
||||
q
|
||||
|> group_by([i], [i.operating_system_version])
|
||||
|> select_merge([i], %{
|
||||
^key =>
|
||||
selected_as(
|
||||
fragment(
|
||||
"if(empty(?), ?, ?)",
|
||||
i.operating_system_version,
|
||||
@not_set,
|
||||
i.operating_system_version
|
||||
),
|
||||
^key
|
||||
)
|
||||
})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, dim, key) when dim in [:entry_page, :exit_page] do
|
||||
q
|
||||
|> group_by([i], field(i, ^dim))
|
||||
|> select_merge([i], %{^shortname(query, dim) => field(i, ^dim)})
|
||||
|> select_merge([i], %{^key => selected_as(field(i, ^dim), ^key)})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, :name, query) do
|
||||
defp group_imported_by(q, :name, key) do
|
||||
q
|
||||
|> group_by([i], i.name)
|
||||
|> select_merge([i], %{^shortname(query, :name) => i.name})
|
||||
|> select_merge([i], %{^key => selected_as(i.name, ^key)})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, :url, query) when query.v2 do
|
||||
defp group_imported_by(q, :url, key) do
|
||||
q
|
||||
|> group_by([i], i.link_url)
|
||||
|> select_merge([i], %{
|
||||
^shortname(query, :url) => fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)
|
||||
^key => selected_as(fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none), ^key)
|
||||
})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, :url, _query) do
|
||||
q
|
||||
|> group_by([i], i.link_url)
|
||||
|> select_merge([i], %{
|
||||
breakdown_prop_value: fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)
|
||||
})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, :path, query) when query.v2 do
|
||||
defp group_imported_by(q, :path, key) do
|
||||
q
|
||||
|> group_by([i], i.path)
|
||||
|> select_merge([i], %{
|
||||
^shortname(query, :path) => fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)
|
||||
^key => selected_as(fragment("if(not empty(?), ?, ?)", i.path, i.path, @none), ^key)
|
||||
})
|
||||
end
|
||||
|
||||
defp group_imported_by(q, :path, _query) do
|
||||
q
|
||||
|> group_by([i], i.path)
|
||||
|> select_merge([i], %{
|
||||
breakdown_prop_value: fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)
|
||||
})
|
||||
defp select_joined_dimensions(q, query) do
|
||||
Enum.reduce(query.dimensions, q, fn dimension, q ->
|
||||
select_joined_dimension(q, dimension, shortname(query, dimension))
|
||||
end)
|
||||
end
|
||||
|
||||
defp select_joined_dimension(q, :city, query) do
|
||||
defp select_joined_dimension(q, "visit:city", key) do
|
||||
select_merge(q, [s, i], %{
|
||||
^shortname(query, :city) => fragment("greatest(?,?)", i.city, s.city)
|
||||
^key => selected_as(fragment("greatest(?,?)", field(i, ^key), field(s, ^key)), ^key)
|
||||
})
|
||||
end
|
||||
|
||||
defp select_joined_dimension(q, :os_version, query) when not query.v2 do
|
||||
defp select_joined_dimension(q, _dimension, key) do
|
||||
select_merge(q, [s, i], %{
|
||||
os: fragment("if(empty(?), ?, ?)", s.os, i.os, s.os),
|
||||
os_version: fragment("if(empty(?), ?, ?)", s.os_version, i.os_version, s.os_version)
|
||||
})
|
||||
end
|
||||
|
||||
defp select_joined_dimension(q, :browser_version, query) when not query.v2 do
|
||||
select_merge(q, [s, i], %{
|
||||
browser: fragment("if(empty(?), ?, ?)", s.browser, i.browser, s.browser),
|
||||
browser_version:
|
||||
fragment("if(empty(?), ?, ?)", s.browser_version, i.browser_version, s.browser_version)
|
||||
})
|
||||
end
|
||||
|
||||
defp select_joined_dimension(q, dim, query) when dim in [:url, :path] and not query.v2 do
|
||||
select_merge(q, [s, i], %{
|
||||
breakdown_prop_value:
|
||||
fragment(
|
||||
"if(empty(?), ?, ?)",
|
||||
s.breakdown_prop_value,
|
||||
i.breakdown_prop_value,
|
||||
s.breakdown_prop_value
|
||||
)
|
||||
})
|
||||
end
|
||||
|
||||
defp select_joined_dimension(q, dim, query) do
|
||||
select_merge(q, [s, i], %{
|
||||
^shortname(query, dim) =>
|
||||
fragment(
|
||||
"if(empty(?), ?, ?)",
|
||||
field(s, ^shortname(query, dim)),
|
||||
field(i, ^shortname(query, dim)),
|
||||
field(s, ^shortname(query, dim))
|
||||
^key =>
|
||||
selected_as(
|
||||
fragment(
|
||||
"if(empty(?), ?, ?)",
|
||||
field(s, ^key),
|
||||
field(i, ^key),
|
||||
field(s, ^key)
|
||||
),
|
||||
^key
|
||||
)
|
||||
})
|
||||
end
|
||||
@ -882,21 +827,12 @@ defmodule Plausible.Stats.Imported do
|
||||
|> select_joined_metrics(rest)
|
||||
end
|
||||
|
||||
defp apply_order_by(q, %Query{v2: true}, _), do: q
|
||||
|
||||
defp apply_order_by(q, query, [:visitors | rest]) do
|
||||
order_by(q, [s, i], desc: s.visitors + i.visitors)
|
||||
|> apply_order_by(query, rest)
|
||||
end
|
||||
|
||||
defp apply_order_by(q, _query, _), do: q
|
||||
|
||||
defp naive_dimension_join(q1, q2, metrics) do
|
||||
from(a in Ecto.Query.subquery(q1),
|
||||
from(a in subquery(q1),
|
||||
full_join: b in subquery(q2),
|
||||
on: a.dim0 == b.dim0,
|
||||
select: %{
|
||||
dim0: fragment("coalesce(?, ?)", a.dim0, b.dim0)
|
||||
dim0: selected_as(fragment("if(? != 0, ?, ?)", a.dim0, a.dim0, b.dim0), :dim0)
|
||||
}
|
||||
)
|
||||
|> select_joined_metrics(metrics)
|
||||
|
@ -17,7 +17,8 @@ defmodule Plausible.Stats.Metrics do
|
||||
:events,
|
||||
:conversion_rate,
|
||||
:group_conversion_rate,
|
||||
:time_on_page
|
||||
:time_on_page,
|
||||
:percentage
|
||||
] ++ on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: [])
|
||||
|
||||
@metric_mappings Enum.into(@all_metrics, %{}, fn metric -> {to_string(metric), metric} end)
|
||||
@ -27,4 +28,8 @@ defmodule Plausible.Stats.Metrics do
|
||||
def from_string!(str) do
|
||||
Map.fetch!(@metric_mappings, str)
|
||||
end
|
||||
|
||||
def from_string(str) do
|
||||
Map.fetch(@metric_mappings, str)
|
||||
end
|
||||
end
|
||||
|
@ -225,6 +225,12 @@ defmodule Plausible.Stats.Query do
|
||||
struct!(query, filters: Filters.parse(params["filters"]))
|
||||
end
|
||||
|
||||
def set(query, keywords) do
|
||||
query
|
||||
|> struct!(keywords)
|
||||
|> refresh_imported_opts()
|
||||
end
|
||||
|
||||
@spec set_dimensions(t(), list(String.t())) :: t()
|
||||
def set_dimensions(query, dimensions) do
|
||||
query
|
||||
|
@ -3,6 +3,7 @@ defmodule Plausible.Stats.QueryOptimizer do
|
||||
Methods to manipulate Query for business logic reasons before building an ecto query.
|
||||
"""
|
||||
|
||||
use Plausible
|
||||
alias Plausible.Stats.{Query, TableDecider, Util}
|
||||
|
||||
@doc """
|
||||
@ -148,8 +149,17 @@ defmodule Plausible.Stats.QueryOptimizer do
|
||||
dimension -> dimension
|
||||
end)
|
||||
|
||||
query
|
||||
|> Query.set_metrics(session_metrics)
|
||||
|> Query.set_dimensions(dimensions)
|
||||
filters =
|
||||
if "event:page" in query.dimensions do
|
||||
query.filters
|
||||
|> Enum.map(fn
|
||||
[op, "event:page" | rest] -> [op, "visit:entry_page" | rest]
|
||||
filter -> filter
|
||||
end)
|
||||
else
|
||||
query.filters
|
||||
end
|
||||
|
||||
Query.set(query, filters: filters, metrics: session_metrics, dimensions: dimensions)
|
||||
end
|
||||
end
|
||||
|
@ -11,84 +11,121 @@ defmodule Plausible.Stats.SQL.Expression do
|
||||
@no_ref "Direct / None"
|
||||
@not_set "(not set)"
|
||||
|
||||
defmacrop field_or_blank_value(expr, empty_value) do
|
||||
defmacrop field_or_blank_value(expr, empty_value, select_alias) do
|
||||
quote do
|
||||
dynamic(
|
||||
[t],
|
||||
fragment("if(empty(?), ?, ?)", unquote(expr), unquote(empty_value), unquote(expr))
|
||||
selected_as(
|
||||
fragment("if(empty(?), ?, ?)", unquote(expr), unquote(empty_value), unquote(expr)),
|
||||
^unquote(select_alias)
|
||||
)
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
def dimension("time:hour", query) do
|
||||
dynamic([t], fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone))
|
||||
end
|
||||
|
||||
def dimension("time:day", query) do
|
||||
dynamic([t], fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone))
|
||||
end
|
||||
|
||||
def dimension("time:month", query) do
|
||||
dynamic([t], fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone))
|
||||
end
|
||||
|
||||
def dimension("event:name", _query), do: dynamic([t], t.name)
|
||||
def dimension("event:page", _query), do: dynamic([t], t.pathname)
|
||||
def dimension("event:hostname", _query), do: dynamic([t], t.hostname)
|
||||
|
||||
def dimension("event:props:" <> property_name, _query) do
|
||||
def dimension("time:hour", query, select_alias) do
|
||||
dynamic(
|
||||
[t],
|
||||
fragment(
|
||||
"if(not empty(?), ?, '(none)')",
|
||||
get_by_key(t, :meta, ^property_name),
|
||||
get_by_key(t, :meta, ^property_name)
|
||||
selected_as(
|
||||
fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone),
|
||||
^select_alias
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
def dimension("visit:entry_page", _query), do: dynamic([t], t.entry_page)
|
||||
def dimension("visit:exit_page", _query), do: dynamic([t], t.exit_page)
|
||||
def dimension("time:day", query, select_alias) do
|
||||
dynamic(
|
||||
[t],
|
||||
selected_as(
|
||||
fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone),
|
||||
^select_alias
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
def dimension("visit:utm_medium", _query),
|
||||
do: field_or_blank_value(t.utm_medium, @not_set)
|
||||
def dimension("time:month", query, select_alias) do
|
||||
dynamic(
|
||||
[t],
|
||||
selected_as(
|
||||
fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone),
|
||||
^select_alias
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
def dimension("visit:utm_source", _query),
|
||||
do: field_or_blank_value(t.utm_source, @not_set)
|
||||
def dimension("event:name", _query, select_alias),
|
||||
do: dynamic([t], selected_as(t.name, ^select_alias))
|
||||
|
||||
def dimension("visit:utm_campaign", _query),
|
||||
do: field_or_blank_value(t.utm_campaign, @not_set)
|
||||
def dimension("event:page", _query, select_alias),
|
||||
do: dynamic([t], selected_as(t.pathname, ^select_alias))
|
||||
|
||||
def dimension("visit:utm_content", _query),
|
||||
do: field_or_blank_value(t.utm_content, @not_set)
|
||||
def dimension("event:hostname", _query, select_alias),
|
||||
do: dynamic([t], selected_as(t.hostname, ^select_alias))
|
||||
|
||||
def dimension("visit:utm_term", _query),
|
||||
do: field_or_blank_value(t.utm_term, @not_set)
|
||||
def dimension("event:props:" <> property_name, _query, select_alias) do
|
||||
dynamic(
|
||||
[t],
|
||||
selected_as(
|
||||
fragment(
|
||||
"if(not empty(?), ?, '(none)')",
|
||||
get_by_key(t, :meta, ^property_name),
|
||||
get_by_key(t, :meta, ^property_name)
|
||||
),
|
||||
^select_alias
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
def dimension("visit:source", _query),
|
||||
do: field_or_blank_value(t.source, @no_ref)
|
||||
def dimension("visit:entry_page", _query, select_alias),
|
||||
do: dynamic([t], selected_as(t.entry_page, ^select_alias))
|
||||
|
||||
def dimension("visit:referrer", _query),
|
||||
do: field_or_blank_value(t.referrer, @no_ref)
|
||||
def dimension("visit:exit_page", _query, select_alias),
|
||||
do: dynamic([t], selected_as(t.exit_page, ^select_alias))
|
||||
|
||||
def dimension("visit:device", _query),
|
||||
do: field_or_blank_value(t.device, @not_set)
|
||||
def dimension("visit:utm_medium", _query, select_alias),
|
||||
do: field_or_blank_value(t.utm_medium, @not_set, select_alias)
|
||||
|
||||
def dimension("visit:os", _query), do: field_or_blank_value(t.os, @not_set)
|
||||
def dimension("visit:utm_source", _query, select_alias),
|
||||
do: field_or_blank_value(t.utm_source, @not_set, select_alias)
|
||||
|
||||
def dimension("visit:os_version", _query),
|
||||
do: field_or_blank_value(t.os_version, @not_set)
|
||||
def dimension("visit:utm_campaign", _query, select_alias),
|
||||
do: field_or_blank_value(t.utm_campaign, @not_set, select_alias)
|
||||
|
||||
def dimension("visit:browser", _query),
|
||||
do: field_or_blank_value(t.browser, @not_set)
|
||||
def dimension("visit:utm_content", _query, select_alias),
|
||||
do: field_or_blank_value(t.utm_content, @not_set, select_alias)
|
||||
|
||||
def dimension("visit:browser_version", _query),
|
||||
do: field_or_blank_value(t.browser_version, @not_set)
|
||||
def dimension("visit:utm_term", _query, select_alias),
|
||||
do: field_or_blank_value(t.utm_term, @not_set, select_alias)
|
||||
|
||||
# :TODO: Locations also set extra filters
|
||||
def dimension("visit:country", _query), do: dynamic([t], t.country)
|
||||
def dimension("visit:region", _query), do: dynamic([t], t.region)
|
||||
def dimension("visit:city", _query), do: dynamic([t], t.city)
|
||||
def dimension("visit:source", _query, select_alias),
|
||||
do: field_or_blank_value(t.source, @no_ref, select_alias)
|
||||
|
||||
def dimension("visit:referrer", _query, select_alias),
|
||||
do: field_or_blank_value(t.referrer, @no_ref, select_alias)
|
||||
|
||||
def dimension("visit:device", _query, select_alias),
|
||||
do: field_or_blank_value(t.device, @not_set, select_alias)
|
||||
|
||||
def dimension("visit:os", _query, select_alias),
|
||||
do: field_or_blank_value(t.os, @not_set, select_alias)
|
||||
|
||||
def dimension("visit:os_version", _query, select_alias),
|
||||
do: field_or_blank_value(t.os_version, @not_set, select_alias)
|
||||
|
||||
def dimension("visit:browser", _query, select_alias),
|
||||
do: field_or_blank_value(t.browser, @not_set, select_alias)
|
||||
|
||||
def dimension("visit:browser_version", _query, select_alias),
|
||||
do: field_or_blank_value(t.browser_version, @not_set, select_alias)
|
||||
|
||||
def dimension("visit:country", _query, select_alias),
|
||||
do: dynamic([t], selected_as(t.country, ^select_alias))
|
||||
|
||||
def dimension("visit:region", _query, select_alias),
|
||||
do: dynamic([t], selected_as(t.region, ^select_alias))
|
||||
|
||||
def dimension("visit:city", _query, select_alias),
|
||||
do: dynamic([t], selected_as(t.city, ^select_alias))
|
||||
|
||||
defmacro event_goal_join(events, page_regexes) do
|
||||
quote do
|
||||
|
@ -7,7 +7,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
|
||||
import Plausible.Stats.Imported
|
||||
import Plausible.Stats.Util
|
||||
|
||||
alias Plausible.Stats.{Base, Query, QueryOptimizer, TableDecider, Filters, Metrics}
|
||||
alias Plausible.Stats.{Base, Query, QueryOptimizer, TableDecider, Filters}
|
||||
alias Plausible.Stats.SQL.Expression
|
||||
|
||||
require Plausible.Stats.SQL.Expression
|
||||
@ -44,6 +44,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
|
||||
|> merge_imported(site, events_query, events_query.metrics)
|
||||
|> maybe_add_global_conversion_rate(site, events_query)
|
||||
|> maybe_add_group_conversion_rate(site, events_query)
|
||||
|> Base.add_percentage_metric(site, events_query, events_query.metrics)
|
||||
end
|
||||
|
||||
defp join_sessions_if_needed(q, site, query) do
|
||||
@ -84,6 +85,9 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
|
||||
|> join_events_if_needed(site, sessions_query)
|
||||
|> build_group_by(sessions_query)
|
||||
|> merge_imported(site, sessions_query, sessions_query.metrics)
|
||||
|> maybe_add_global_conversion_rate(site, sessions_query)
|
||||
|> maybe_add_group_conversion_rate(site, sessions_query)
|
||||
|> Base.add_percentage_metric(site, sessions_query, sessions_query.metrics)
|
||||
end
|
||||
|
||||
def join_events_if_needed(q, site, query) do
|
||||
@ -123,36 +127,23 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
|
||||
^shortname(query, dimension) => fragment("?", goal)
|
||||
},
|
||||
group_by: goal,
|
||||
where: goal != 0
|
||||
where: goal != 0 and (e.name == "pageview" or goal < 0)
|
||||
)
|
||||
end
|
||||
|
||||
defp dimension_group_by(q, query, dimension) do
|
||||
key = shortname(query, dimension)
|
||||
|
||||
q
|
||||
|> select_merge(^%{shortname(query, dimension) => Expression.dimension(dimension, query)})
|
||||
|> group_by(^Expression.dimension(dimension, query))
|
||||
|> select_merge(^%{key => Expression.dimension(dimension, query, key)})
|
||||
|> group_by([], selected_as(^key))
|
||||
end
|
||||
|
||||
defp build_order_by(q, query, mode) do
|
||||
Enum.reduce(query.order_by || [], q, &build_order_by(&2, query, &1, mode))
|
||||
defp build_order_by(q, query) do
|
||||
Enum.reduce(query.order_by || [], q, &build_order_by(&2, query, &1))
|
||||
end
|
||||
|
||||
def build_order_by(q, query, {metric_or_dimension, order_direction}, :inner) do
|
||||
order_by(
|
||||
q,
|
||||
[t],
|
||||
^{
|
||||
order_direction,
|
||||
if(
|
||||
Metrics.metric?(metric_or_dimension),
|
||||
do: dynamic([], selected_as(^shortname(query, metric_or_dimension))),
|
||||
else: Expression.dimension(metric_or_dimension, query)
|
||||
)
|
||||
}
|
||||
)
|
||||
end
|
||||
|
||||
def build_order_by(q, query, {metric_or_dimension, order_direction}, :outer) do
|
||||
def build_order_by(q, query, {metric_or_dimension, order_direction}) do
|
||||
order_by(
|
||||
q,
|
||||
[t],
|
||||
@ -262,10 +253,10 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
|
||||
defp join_query_results({nil, _}, {nil, _}), do: nil
|
||||
|
||||
defp join_query_results({events_q, events_query}, {nil, _}),
|
||||
do: events_q |> build_order_by(events_query, :inner)
|
||||
do: events_q |> build_order_by(events_query)
|
||||
|
||||
defp join_query_results({nil, _}, {sessions_q, sessions_query}),
|
||||
do: sessions_q |> build_order_by(sessions_query, :inner)
|
||||
defp join_query_results({nil, events_query}, {sessions_q, _}),
|
||||
do: sessions_q |> build_order_by(events_query)
|
||||
|
||||
defp join_query_results({events_q, events_query}, {sessions_q, sessions_query}) do
|
||||
join(subquery(events_q), :left, [e], s in subquery(sessions_q),
|
||||
@ -274,12 +265,12 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
|
||||
|> select_join_fields(events_query, events_query.dimensions, e)
|
||||
|> select_join_fields(events_query, events_query.metrics, e)
|
||||
|> select_join_fields(sessions_query, List.delete(sessions_query.metrics, :sample_percent), s)
|
||||
|> build_order_by(events_query, :outer)
|
||||
|> build_order_by(events_query)
|
||||
end
|
||||
|
||||
defp build_group_by_join(%Query{dimensions: []}), do: true
|
||||
def build_group_by_join(%Query{dimensions: []}), do: true
|
||||
|
||||
defp build_group_by_join(query) do
|
||||
def build_group_by_join(query) do
|
||||
query.dimensions
|
||||
|> Enum.map(fn dim ->
|
||||
dynamic([e, s], field(e, ^shortname(query, dim)) == field(s, ^shortname(query, dim)))
|
||||
|
@ -29,34 +29,42 @@ defmodule Plausible.Stats.TableDecider do
|
||||
|> filter_keys()
|
||||
|> partition(query, &filters_partitioner/2)
|
||||
|
||||
%{event: event_only_dimensions, session: session_only_dimensions} =
|
||||
partition(query.dimensions, query, &filters_partitioner/2)
|
||||
|
||||
cond do
|
||||
# Only one table needs to be queried
|
||||
empty?(event_only_metrics) && empty?(event_only_filters) ->
|
||||
empty?(event_only_metrics) && empty?(event_only_filters) && empty?(event_only_dimensions) ->
|
||||
{[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}
|
||||
|
||||
empty?(session_only_metrics) && empty?(session_only_filters) ->
|
||||
empty?(session_only_metrics) && empty?(session_only_filters) &&
|
||||
empty?(session_only_dimensions) ->
|
||||
{event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics}
|
||||
|
||||
# Filters on both events and sessions, but only one kind of metric
|
||||
empty?(event_only_metrics) ->
|
||||
# Filters and/or dimensions on both events and sessions, but only one kind of metric
|
||||
empty?(event_only_metrics) && empty?(event_only_dimensions) ->
|
||||
{[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}
|
||||
|
||||
empty?(session_only_metrics) ->
|
||||
empty?(session_only_metrics) && empty?(session_only_dimensions) ->
|
||||
{event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics}
|
||||
|
||||
# Default: prefer sessions
|
||||
# Default: prefer events
|
||||
true ->
|
||||
{event_only_metrics ++ sample_percent,
|
||||
session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}
|
||||
{event_only_metrics ++ either_metrics ++ sample_percent,
|
||||
session_only_metrics ++ sample_percent, other_metrics}
|
||||
end
|
||||
end
|
||||
|
||||
defp filter_keys(query) do
|
||||
query.filters
|
||||
|> Enum.map(fn [_, filter_key | _rest] -> filter_key end)
|
||||
|> Enum.concat(query.dimensions)
|
||||
end
|
||||
|
||||
defp metric_partitioner(%Query{v2: true}, :conversion_rate), do: :either
|
||||
defp metric_partitioner(%Query{v2: true}, :group_conversion_rate), do: :either
|
||||
defp metric_partitioner(%Query{v2: true}, :visitors), do: :either
|
||||
defp metric_partitioner(%Query{v2: true}, :visits), do: :either
|
||||
|
||||
defp metric_partitioner(_, :conversion_rate), do: :event
|
||||
defp metric_partitioner(_, :group_conversion_rate), do: :event
|
||||
defp metric_partitioner(_, :average_revenue), do: :event
|
||||
@ -76,7 +84,7 @@ defmodule Plausible.Stats.TableDecider do
|
||||
# Calculated metrics - handled on callsite separately from other metrics.
|
||||
defp metric_partitioner(_, :time_on_page), do: :other
|
||||
defp metric_partitioner(_, :total_visitors), do: :other
|
||||
defp metric_partitioner(_, :percentage), do: :other
|
||||
defp metric_partitioner(_, :percentage), do: :either
|
||||
# Sample percentage is included in both tables if queried.
|
||||
defp metric_partitioner(_, :sample_percent), do: :sample_percent
|
||||
|
||||
|
@ -622,7 +622,7 @@ defmodule Plausible.Imported.CSVImporterTest do
|
||||
case params_or_site do
|
||||
%Plausible.Site{} = site ->
|
||||
common_params.(site)
|
||||
|> Map.put("metrics", "visitors,visits,pageviews,visit_duration,bounce_rate")
|
||||
|> Map.put("metrics", "visitors,visits,visit_duration,bounce_rate")
|
||||
|> Map.put("limit", 1000)
|
||||
|> Map.put("property", by)
|
||||
|
||||
@ -669,7 +669,7 @@ defmodule Plausible.Imported.CSVImporterTest do
|
||||
assert exported["pageviews"] == imported["pageviews"]
|
||||
assert exported["bounce_rate"] == imported["bounce_rate"]
|
||||
assert_in_delta exported["visitors"], imported["visitors"], 1
|
||||
assert exported["visits"] == imported["visits"]
|
||||
assert_in_delta exported["visits"], imported["visits"], 1
|
||||
assert_in_delta exported["visit_duration"], imported["visit_duration"], 1
|
||||
end)
|
||||
|
||||
@ -810,7 +810,7 @@ defmodule Plausible.Imported.CSVImporterTest do
|
||||
_no_diff = 0
|
||||
end
|
||||
end)
|
||||
) == [0.0, 0.0, 0.0, 0.0, 0.03614457831325302]
|
||||
) == [0.0, 0.0, 0.0, 0.0, 0.0]
|
||||
|
||||
# NOTE: city breakdown's visit duration difference is up to 14%
|
||||
assert summary(field(exported_cities, "visit_duration")) == [0, 0.0, 0.0, 1.0, 1718]
|
||||
@ -829,7 +829,7 @@ defmodule Plausible.Imported.CSVImporterTest do
|
||||
_no_diff = 0
|
||||
end
|
||||
end)
|
||||
) == [0, 0.0, 0.0, 0.0, 0.1428571428571429]
|
||||
) == [0, 0.0, 0.0, 0.0, 0.0]
|
||||
|
||||
# NOTE: city breakdown's visitors relative difference is up to 27%
|
||||
assert summary(field(exported_cities, "visitors")) == [1, 1.0, 1.0, 2.0, 22]
|
||||
|
@ -117,7 +117,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
|
||||
|
||||
breakdown_params =
|
||||
common_params
|
||||
|> Map.put("metrics", "visitors,visits,pageviews,visit_duration,bounce_rate")
|
||||
|> Map.put("metrics", "visitors,visits,visit_duration,bounce_rate")
|
||||
|> Map.put("limit", 1000)
|
||||
|
||||
%{key: api_key} = insert(:api_key, user: user)
|
||||
@ -467,7 +467,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
|
||||
|
||||
assert List.first(results) == %{
|
||||
"bounce_rate" => 35.0,
|
||||
"pageviews" => 6229,
|
||||
"visit_duration" => 40.0,
|
||||
"visitors" => 4671,
|
||||
"visits" => 4917,
|
||||
@ -475,12 +474,11 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
|
||||
}
|
||||
|
||||
assert List.last(results) == %{
|
||||
"bounce_rate" => 100.0,
|
||||
"pageviews" => 1,
|
||||
"visit_duration" => 0.0,
|
||||
"bounce_rate" => 0.0,
|
||||
"source" => "yahoo",
|
||||
"visit_duration" => 41.0,
|
||||
"visitors" => 1,
|
||||
"visits" => 1,
|
||||
"source" => "petalsearch.com"
|
||||
"visits" => 1
|
||||
}
|
||||
end
|
||||
|
||||
@ -493,7 +491,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
|
||||
assert [
|
||||
%{
|
||||
"bounce_rate" => 35.0,
|
||||
"pageviews" => 6399,
|
||||
"utm_medium" => "organic",
|
||||
"visit_duration" => 40.0,
|
||||
"visitors" => 4787,
|
||||
@ -501,7 +498,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
|
||||
},
|
||||
%{
|
||||
"bounce_rate" => 58.0,
|
||||
"pageviews" => 491,
|
||||
"utm_medium" => "referral",
|
||||
"visit_duration" => 27.0,
|
||||
"visitors" => 294,
|
||||
@ -520,7 +516,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
|
||||
|
||||
assert List.first(results) == %{
|
||||
"bounce_rate" => 35.0,
|
||||
"pageviews" => 838,
|
||||
"visit_duration" => 43.0,
|
||||
"visitors" => 675,
|
||||
"visits" => 712,
|
||||
@ -529,11 +524,10 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
|
||||
|
||||
assert List.last(results) == %{
|
||||
"bounce_rate" => 0.0,
|
||||
"pageviews" => 1,
|
||||
"visit_duration" => 27.0,
|
||||
"entry_page" => "/znamenitosti-rima-koje-treba-vidjeti",
|
||||
"visit_duration" => 40.0,
|
||||
"visitors" => 1,
|
||||
"visits" => 1,
|
||||
"entry_page" => "/kad-lisce-pada"
|
||||
"visits" => 1
|
||||
}
|
||||
end
|
||||
|
||||
@ -543,12 +537,11 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
|
||||
%{"results" => results} =
|
||||
get(conn, "/api/v1/stats/breakdown", params) |> json_response(200)
|
||||
|
||||
assert length(results) == 488
|
||||
assert length(results) == 494
|
||||
|
||||
assert List.first(results) == %{
|
||||
"bounce_rate" => 35.0,
|
||||
"city" => 792_680,
|
||||
"pageviews" => 1650,
|
||||
"visit_duration" => 39.0,
|
||||
"visitors" => 1233,
|
||||
"visits" => 1273
|
||||
@ -556,9 +549,8 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
|
||||
|
||||
assert List.last(results) == %{
|
||||
"bounce_rate" => 0.0,
|
||||
"city" => 4_399_605,
|
||||
"pageviews" => 7,
|
||||
"visit_duration" => 128.0,
|
||||
"city" => 11_951_298,
|
||||
"visit_duration" => 271.0,
|
||||
"visitors" => 1,
|
||||
"visits" => 1
|
||||
}
|
||||
@ -574,7 +566,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
|
||||
|
||||
assert List.first(results) == %{
|
||||
"bounce_rate" => 38.0,
|
||||
"pageviews" => 7041,
|
||||
"visit_duration" => 37.0,
|
||||
"visitors" => 5277,
|
||||
"visits" => 5532,
|
||||
@ -583,7 +574,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
|
||||
|
||||
assert List.last(results) == %{
|
||||
"bounce_rate" => 37.0,
|
||||
"pageviews" => 143,
|
||||
"visit_duration" => 60.0,
|
||||
"visitors" => 97,
|
||||
"visits" => 100,
|
||||
@ -601,7 +591,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
|
||||
|
||||
assert List.first(results) == %{
|
||||
"bounce_rate" => 33.0,
|
||||
"pageviews" => 8143,
|
||||
"visit_duration" => 50.0,
|
||||
"visitors" => 4625,
|
||||
"visits" => 4655,
|
||||
@ -610,7 +599,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
|
||||
|
||||
assert List.last(results) == %{
|
||||
"bounce_rate" => 0.0,
|
||||
"pageviews" => 6,
|
||||
"visit_duration" => 0.0,
|
||||
"visitors" => 1,
|
||||
"visits" => 1,
|
||||
@ -628,7 +616,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
|
||||
|
||||
assert List.first(results) == %{
|
||||
"bounce_rate" => 34.0,
|
||||
"pageviews" => 5827,
|
||||
"visit_duration" => 41.0,
|
||||
"visitors" => 4319,
|
||||
"visits" => 4495,
|
||||
@ -637,7 +624,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
|
||||
|
||||
assert List.last(results) == %{
|
||||
"bounce_rate" => 0.0,
|
||||
"pageviews" => 6,
|
||||
"visit_duration" => 0.0,
|
||||
"visitors" => 1,
|
||||
"visits" => 1,
|
||||
@ -657,7 +643,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
|
||||
"bounce_rate" => 32.0,
|
||||
"os" => "Android",
|
||||
"os_version" => "13.0.0",
|
||||
"pageviews" => 1673,
|
||||
"visit_duration" => 42.0,
|
||||
"visitors" => 1247,
|
||||
"visits" => 1295
|
||||
@ -665,17 +650,16 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
|
||||
|
||||
assert List.last(results) == %{
|
||||
"bounce_rate" => 0.0,
|
||||
"os" => "iOS",
|
||||
"os_version" => "15.1",
|
||||
"pageviews" => 1,
|
||||
"visit_duration" => 54.0,
|
||||
"os" => "Chrome OS",
|
||||
"os_version" => "x86_64 15662.76.0",
|
||||
"visit_duration" => 16.0,
|
||||
"visitors" => 1,
|
||||
"visits" => 1
|
||||
}
|
||||
end
|
||||
|
||||
defp assert_pages(conn, params) do
|
||||
metrics = "visitors,visits,pageviews,time_on_page,visit_duration,bounce_rate"
|
||||
metrics = "visitors,visits,time_on_page,visit_duration,bounce_rate"
|
||||
|
||||
params =
|
||||
params
|
||||
@ -686,7 +670,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
|
||||
%{"results" => results} =
|
||||
get(conn, "/api/v1/stats/breakdown", params) |> json_response(200)
|
||||
|
||||
assert length(results) == 729
|
||||
assert length(results) == 730
|
||||
|
||||
# The `event:page` breakdown is currently using the `entry_page`
|
||||
# property to allow querying session metrics.
|
||||
@ -696,7 +680,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
|
||||
# it will allow us to assert on the session metrics as well.
|
||||
assert Enum.at(results, 2) == %{
|
||||
"page" => "/",
|
||||
"pageviews" => 5537,
|
||||
"time_on_page" => 17.677262055264585,
|
||||
"visitors" => 371,
|
||||
"visits" => 212,
|
||||
@ -707,13 +690,12 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
|
||||
# This page was never an entry_page in the imported data, and
|
||||
# therefore the session metrics are returned as `nil`.
|
||||
assert List.last(results) == %{
|
||||
"page" => "/5-dobrih-razloga-zasto-zapoceti-dan-zobenom-kasom/",
|
||||
"pageviews" => 2,
|
||||
"time_on_page" => 10.0,
|
||||
"bounce_rate" => 0.0,
|
||||
"page" => "/znamenitosti-rima-koje-treba-vidjeti/",
|
||||
"time_on_page" => 40.0,
|
||||
"visit_duration" => 0.0,
|
||||
"visitors" => 1,
|
||||
"visits" => 1,
|
||||
"bounce_rate" => nil,
|
||||
"visit_duration" => nil
|
||||
"visits" => 1
|
||||
}
|
||||
end
|
||||
end
|
||||
|
@ -76,7 +76,7 @@ defmodule Plausible.Stats.TableDeciderTest do
|
||||
query = make_query(false, [])
|
||||
|
||||
assert partition_metrics([:time_on_page, :percentage, :total_visitors], query) ==
|
||||
{[], [], [:time_on_page, :percentage, :total_visitors]}
|
||||
{[], [:percentage], [:time_on_page, :total_visitors]}
|
||||
end
|
||||
|
||||
test "raises if unknown metric" do
|
||||
@ -108,11 +108,11 @@ defmodule Plausible.Stats.TableDeciderTest do
|
||||
{[], [:visit_duration, :visits], []}
|
||||
end
|
||||
|
||||
test "metrics that can be calculated on either are biased to sessions" do
|
||||
test "metrics that can be calculated on either are biased to events" do
|
||||
query = make_query(true, [])
|
||||
|
||||
assert partition_metrics([:bounce_rate, :total_revenue, :visitors], query) ==
|
||||
{[:total_revenue], [:bounce_rate, :visitors], []}
|
||||
{[:total_revenue, :visitors], [:bounce_rate], []}
|
||||
end
|
||||
|
||||
test "sample_percent is handled with either metrics" do
|
||||
|
@ -1,2 +1,2 @@
|
||||
name,visitors,pageviews,bounce_rate,time_on_page
|
||||
/some-other-page,1,1,,60.0
|
||||
/some-other-page,1,1,0,60.0
|
||||
|
|
@ -1,4 +1,4 @@
|
||||
name,visitors,pageviews,bounce_rate,time_on_page
|
||||
/,4,3,67,
|
||||
/signup,1,1,0,60.0
|
||||
/some-other-page,1,1,,60.0
|
||||
/some-other-page,1,1,0,60.0
|
||||
|
|
@ -1,4 +1,4 @@
|
||||
name,visitors,pageviews,bounce_rate,time_on_page
|
||||
/,5,4,75,
|
||||
/signup,1,1,0,60.0
|
||||
/some-other-page,1,1,,60.0
|
||||
/some-other-page,1,1,0,60.0
|
||||
|
|
@ -890,8 +890,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
|
||||
assert json_response(conn, 200) == %{
|
||||
"results" => [
|
||||
%{"page" => "/", "pageviews" => 2},
|
||||
%{"page" => "/plausible.io", "pageviews" => 1},
|
||||
%{"page" => "/include-me", "pageviews" => 1}
|
||||
%{"page" => "/include-me", "pageviews" => 1},
|
||||
%{"page" => "/plausible.io", "pageviews" => 1}
|
||||
]
|
||||
}
|
||||
end
|
||||
@ -1023,7 +1023,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
|
||||
"period" => "day",
|
||||
"date" => "2021-01-01",
|
||||
"property" => "visit:exit_page",
|
||||
"metrics" => "visitors,visits,pageviews,bounce_rate,visit_duration,events",
|
||||
"metrics" => "visitors,visits,bounce_rate,visit_duration",
|
||||
"with_imported" => "true"
|
||||
})
|
||||
|
||||
@ -1031,18 +1031,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
|
||||
"results" => [
|
||||
%{
|
||||
"bounce_rate" => 0.0,
|
||||
"events" => 7,
|
||||
"exit_page" => "/b",
|
||||
"pageviews" => 7,
|
||||
"visit_duration" => 150.0,
|
||||
"visitors" => 3,
|
||||
"visits" => 4
|
||||
},
|
||||
%{
|
||||
"bounce_rate" => 100.0,
|
||||
"events" => 1,
|
||||
"exit_page" => "/a",
|
||||
"pageviews" => 1,
|
||||
"visit_duration" => 0.0,
|
||||
"visitors" => 1,
|
||||
"visits" => 1
|
||||
@ -2176,8 +2172,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
|
||||
|
||||
assert json_response(conn, 200) == %{
|
||||
"results" => [
|
||||
%{"page" => "/plausible.io", "bounce_rate" => 100},
|
||||
%{"page" => "/important-page", "bounce_rate" => 100}
|
||||
%{"page" => "/important-page", "bounce_rate" => 100},
|
||||
%{"page" => "/plausible.io", "bounce_rate" => 100}
|
||||
]
|
||||
}
|
||||
end
|
||||
@ -2596,14 +2592,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
|
||||
|
||||
assert json_response(conn, 200) == %{
|
||||
"results" => [
|
||||
%{
|
||||
"page" => "/B",
|
||||
"time_on_page" => 90.0
|
||||
},
|
||||
%{
|
||||
"page" => "/A",
|
||||
"time_on_page" => 60.0
|
||||
},
|
||||
%{
|
||||
"page" => "/B",
|
||||
"time_on_page" => 90.0
|
||||
},
|
||||
%{
|
||||
"page" => "/C",
|
||||
"time_on_page" => nil
|
||||
@ -3045,13 +3041,13 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
|
||||
|
||||
assert json_response(conn, 200) == %{
|
||||
"results" => [
|
||||
%{
|
||||
"entry_page" => "/entry-page-1",
|
||||
"bounce_rate" => 0
|
||||
},
|
||||
%{
|
||||
"entry_page" => "/entry-page-2",
|
||||
"bounce_rate" => 100
|
||||
},
|
||||
%{
|
||||
"entry_page" => "/entry-page-1",
|
||||
"bounce_rate" => 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -3146,6 +3142,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
|
||||
},
|
||||
%{
|
||||
"page" => "/plausible.io",
|
||||
# Breaks for event:page breakdown since visitors is calculated based on entry_page :/
|
||||
"visitors" => 2,
|
||||
"bounce_rate" => 100,
|
||||
"visit_duration" => 0,
|
||||
@ -3290,8 +3287,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
|
||||
|
||||
assert %{"browser" => "Chrome", "events" => 1} = breakdown_and_first.("visit:browser")
|
||||
assert %{"device" => "Desktop", "events" => 1} = breakdown_and_first.("visit:device")
|
||||
assert %{"entry_page" => "/test", "events" => 1} = breakdown_and_first.("visit:entry_page")
|
||||
assert %{"exit_page" => "/test", "events" => 1} = breakdown_and_first.("visit:exit_page")
|
||||
assert %{"country" => "EE", "events" => 1} = breakdown_and_first.("visit:country")
|
||||
assert %{"os" => "Mac", "events" => 1} = breakdown_and_first.("visit:os")
|
||||
assert %{"page" => "/test", "events" => 1} = breakdown_and_first.("event:page")
|
||||
|
@ -1477,7 +1477,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
|
||||
conn =
|
||||
post(conn, "/api/v2/query", %{
|
||||
"site_id" => site.domain,
|
||||
"metrics" => ["visitors"],
|
||||
"metrics" => ["visitors", "percentage"],
|
||||
"date_range" => ["2021-01-01", "2021-01-01"],
|
||||
"dimensions" => [unquote(dimension)]
|
||||
})
|
||||
@ -1485,9 +1485,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
|
||||
%{"results" => results} = json_response(conn, 200)
|
||||
|
||||
assert results == [
|
||||
%{"dimensions" => [unquote(value1)], "metrics" => [3]},
|
||||
%{"dimensions" => [unquote(value2)], "metrics" => [2]},
|
||||
%{"dimensions" => [unquote(blank_value)], "metrics" => [1]}
|
||||
%{"dimensions" => [unquote(value1)], "metrics" => [3, 50]},
|
||||
%{"dimensions" => [unquote(value2)], "metrics" => [2, 33.3]},
|
||||
%{"dimensions" => [unquote(blank_value)], "metrics" => [1, 16.7]}
|
||||
]
|
||||
end
|
||||
end
|
||||
@ -3463,6 +3463,48 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
|
||||
%{"dimensions" => ["Chrome"], "metrics" => [1]}
|
||||
]
|
||||
end
|
||||
|
||||
test "all metrics for breakdown by event prop", %{conn: conn, site: site} do
|
||||
populate_stats(site, [
|
||||
build(:pageview,
|
||||
user_id: 1,
|
||||
pathname: "/",
|
||||
timestamp: ~N[2021-01-01 00:00:00]
|
||||
),
|
||||
build(:pageview,
|
||||
user_id: 1,
|
||||
pathname: "/plausible.io",
|
||||
timestamp: ~N[2021-01-01 00:10:00]
|
||||
),
|
||||
build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),
|
||||
build(:pageview,
|
||||
pathname: "/plausible.io",
|
||||
timestamp: ~N[2021-01-01 00:00:00]
|
||||
)
|
||||
])
|
||||
|
||||
conn =
|
||||
post(conn, "/api/v2/query", %{
|
||||
"site_id" => site.domain,
|
||||
"metrics" => [
|
||||
"visitors",
|
||||
"visits",
|
||||
"pageviews",
|
||||
"events",
|
||||
"bounce_rate",
|
||||
"visit_duration"
|
||||
],
|
||||
"date_range" => "all",
|
||||
"dimensions" => ["event:page"]
|
||||
})
|
||||
|
||||
%{"results" => results} = json_response(conn, 200)
|
||||
|
||||
assert results == [
|
||||
%{"dimensions" => ["/"], "metrics" => [2, 2, 2, 2, 50, 300]},
|
||||
%{"dimensions" => ["/plausible.io"], "metrics" => [2, 2, 2, 2, 100, 0]}
|
||||
]
|
||||
end
|
||||
end
|
||||
|
||||
describe "imported data" do
|
||||
@ -3589,10 +3631,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
|
||||
assert %{"dimensions" => ["Desktop"], "metrics" => [1]} =
|
||||
breakdown_and_first.("visit:device")
|
||||
|
||||
# :TODO: These should not pass validation - not available on events.
|
||||
# visit dimension and event-only metric
|
||||
# assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("visit:entry_page")
|
||||
# assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("visit:exit_page")
|
||||
assert %{"dimensions" => ["EE"], "metrics" => [1]} = breakdown_and_first.("visit:country")
|
||||
assert %{"dimensions" => ["Mac"], "metrics" => [1]} = breakdown_and_first.("visit:os")
|
||||
assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("event:page")
|
||||
@ -3865,6 +3903,56 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
|
||||
|
||||
assert meta["warning"] =~ "Imported stats are not included in the results"
|
||||
end
|
||||
|
||||
test "imported country, region and city data",
|
||||
%{
|
||||
conn: conn,
|
||||
site: site
|
||||
} do
|
||||
site_import = insert(:site_import, site: site)
|
||||
|
||||
populate_stats(site, site_import.id, [
|
||||
build(:pageview,
|
||||
timestamp: ~N[2021-01-01 00:15:00],
|
||||
country_code: "DE",
|
||||
subdivision1_code: "DE-BE",
|
||||
city_geoname_id: 2_950_159
|
||||
),
|
||||
build(:pageview,
|
||||
timestamp: ~N[2021-01-01 00:15:00],
|
||||
country_code: "DE",
|
||||
subdivision1_code: "DE-BE",
|
||||
city_geoname_id: 2_950_159
|
||||
),
|
||||
build(:pageview,
|
||||
timestamp: ~N[2021-01-01 00:15:00],
|
||||
country_code: "EE",
|
||||
subdivision1_code: "EE-37",
|
||||
city_geoname_id: 588_409
|
||||
),
|
||||
build(:imported_locations, country: "EE", region: "EE-37", city: 588_409, visitors: 33)
|
||||
])
|
||||
|
||||
for {dimension, stats_value, imports_value} <- [
|
||||
{"visit:country", "DE", "EE"},
|
||||
{"visit:region", "DE-BE", "EE-37"},
|
||||
{"visit:city", 2_950_159, 588_409}
|
||||
] do
|
||||
conn =
|
||||
post(conn, "/api/v2/query", %{
|
||||
"site_id" => site.domain,
|
||||
"metrics" => ["visitors"],
|
||||
"date_range" => "all",
|
||||
"dimensions" => [dimension],
|
||||
"include" => %{"imports" => true}
|
||||
})
|
||||
|
||||
assert json_response(conn, 200)["results"] == [
|
||||
%{"dimensions" => [imports_value], "metrics" => [34]},
|
||||
%{"dimensions" => [stats_value], "metrics" => [2]}
|
||||
]
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
test "multiple breakdown timeseries with sources", %{conn: conn, site: site} do
|
||||
|
@ -207,6 +207,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
|
||||
]
|
||||
end
|
||||
|
||||
@tag capture_log: true
|
||||
test "garbage filters don't crash the call", %{conn: conn, site: site} do
|
||||
filters =
|
||||
"{\"source\":\"Direct / None\",\"screen\":\"Desktop\",\"browser\":\"Chrome\",\"os\":\"Mac\",\"os_version\":\"10.15\",\"country\":\"DE\",\"city\":\"2950159\"}%' AND 2*3*8=6*8 AND 'L9sv'!='L9sv%"
|
||||
|
@ -601,17 +601,17 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
|
||||
)
|
||||
|
||||
assert json_response(conn, 200)["results"] == [
|
||||
%{
|
||||
"name" => "blog",
|
||||
"visitors" => 2,
|
||||
"bounce_rate" => 50.0,
|
||||
"visit_duration" => 50.0
|
||||
},
|
||||
%{
|
||||
"name" => "ad",
|
||||
"visitors" => 2,
|
||||
"bounce_rate" => 100.0,
|
||||
"visit_duration" => 50.0
|
||||
},
|
||||
%{
|
||||
"name" => "blog",
|
||||
"visitors" => 2,
|
||||
"bounce_rate" => 50.0,
|
||||
"visit_duration" => 50.0
|
||||
}
|
||||
]
|
||||
end
|
||||
@ -708,7 +708,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
|
||||
|
||||
assert json_response(conn, 200)["results"] == [
|
||||
%{
|
||||
"bounce_rate" => nil,
|
||||
"bounce_rate" => 0,
|
||||
"time_on_page" => 60,
|
||||
"visitors" => 3,
|
||||
"pageviews" => 4,
|
||||
|
@ -340,7 +340,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"name" => "/blog/other-post",
|
||||
"visitors" => 1,
|
||||
"pageviews" => 1,
|
||||
"bounce_rate" => nil,
|
||||
"bounce_rate" => 0,
|
||||
"time_on_page" => nil
|
||||
}
|
||||
]
|
||||
@ -392,7 +392,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"name" => "/blog/other-post",
|
||||
"visitors" => 1,
|
||||
"pageviews" => 1,
|
||||
"bounce_rate" => nil,
|
||||
"bounce_rate" => 0,
|
||||
"time_on_page" => nil
|
||||
}
|
||||
]
|
||||
@ -744,7 +744,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"name" => "/blog/post-2",
|
||||
"visitors" => 1,
|
||||
"pageviews" => 1,
|
||||
"bounce_rate" => nil,
|
||||
"bounce_rate" => 0,
|
||||
"time_on_page" => nil
|
||||
}
|
||||
]
|
||||
@ -789,7 +789,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"name" => "/blog/(/post-2",
|
||||
"visitors" => 1,
|
||||
"pageviews" => 1,
|
||||
"bounce_rate" => nil,
|
||||
"bounce_rate" => 0,
|
||||
"time_on_page" => nil
|
||||
}
|
||||
]
|
||||
@ -842,7 +842,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"name" => "/about",
|
||||
"visitors" => 1,
|
||||
"pageviews" => 1,
|
||||
"bounce_rate" => nil,
|
||||
"bounce_rate" => 0,
|
||||
"time_on_page" => nil
|
||||
}
|
||||
]
|
||||
@ -940,7 +940,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"name" => "/"
|
||||
},
|
||||
%{
|
||||
"bounce_rate" => nil,
|
||||
"bounce_rate" => 0,
|
||||
"time_on_page" => nil,
|
||||
"visitors" => 1,
|
||||
"pageviews" => 1,
|
||||
@ -1066,7 +1066,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"visitors" => 2
|
||||
},
|
||||
%{
|
||||
"bounce_rate" => nil,
|
||||
"bounce_rate" => 0,
|
||||
"name" => "/exit-blog",
|
||||
"pageviews" => 1,
|
||||
"time_on_page" => nil,
|
||||
@ -1192,7 +1192,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|
||||
"name" => "/"
|
||||
},
|
||||
%{
|
||||
"bounce_rate" => nil,
|
||||
"bounce_rate" => 0,
|
||||
"time_on_page" => 60,
|
||||
"visitors" => 2,
|
||||
"pageviews" => 2,
|
||||
|
@ -453,6 +453,9 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
|
||||
referrer_source: "DuckDuckGo",
|
||||
referrer: "duckduckgo.com"
|
||||
),
|
||||
build(:imported_sources,
|
||||
source: "DuckDuckGo"
|
||||
),
|
||||
build(:imported_sources,
|
||||
source: "DuckDuckGo"
|
||||
)
|
||||
@ -467,7 +470,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
|
||||
conn = get(conn, "/api/stats/#{site.domain}/sources?limit=1&page=2&with_imported=true")
|
||||
|
||||
assert json_response(conn, 200)["results"] == [
|
||||
%{"name" => "DuckDuckGo", "visitors" => 2}
|
||||
%{"name" => "Google", "visitors" => 2}
|
||||
]
|
||||
end
|
||||
|
||||
@ -590,17 +593,17 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
|
||||
)
|
||||
|
||||
assert json_response(conn, 200)["results"] == [
|
||||
%{
|
||||
"name" => "social",
|
||||
"visitors" => 1,
|
||||
"bounce_rate" => 0,
|
||||
"visit_duration" => 900
|
||||
},
|
||||
%{
|
||||
"name" => "email",
|
||||
"visitors" => 1,
|
||||
"bounce_rate" => 100,
|
||||
"visit_duration" => 0
|
||||
},
|
||||
%{
|
||||
"name" => "social",
|
||||
"visitors" => 1,
|
||||
"bounce_rate" => 0,
|
||||
"visit_duration" => 900
|
||||
}
|
||||
]
|
||||
|
||||
@ -611,17 +614,17 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
|
||||
)
|
||||
|
||||
assert json_response(conn, 200)["results"] == [
|
||||
%{
|
||||
"name" => "social",
|
||||
"visitors" => 2,
|
||||
"bounce_rate" => 50,
|
||||
"visit_duration" => 800.0
|
||||
},
|
||||
%{
|
||||
"name" => "email",
|
||||
"visitors" => 2,
|
||||
"bounce_rate" => 50,
|
||||
"visit_duration" => 50
|
||||
},
|
||||
%{
|
||||
"name" => "social",
|
||||
"visitors" => 2,
|
||||
"bounce_rate" => 50,
|
||||
"visit_duration" => 800.0
|
||||
}
|
||||
]
|
||||
end
|
||||
|
Loading…
Reference in New Issue
Block a user