plausible/analytics
1
1
Fork 0
mirror of https://github.com/plausible/analytics.git synced 2024-12-23 01:22:15 +03:00
Code Issues Packages Projects Releases Wiki Activity

APIv2: macros, SQL cleanup (#4286)

Browse Source 
* Move fragments module under Plausible.Stats.SQL

* Introduce select_merge_as macro

This simplifies some select_merge calls

* Simplify select_join_fields

* Remove a needless dynamic

* wrap_select_columns macro

* Move metrics from base.ex to expression.ex

* Move WhereBuilder under Plausible.Stats.SQL

* Moduledoc

* Improved macros

* Wrap more code

* select_merge_as more

* Move defp to the end

* wrap_alias
This commit is contained in:
Karl-Aksel Puulmann 2024-07-03 16:32:25 +03:00 committed by GitHub
parent 790984e1ad
commit 05ac840078
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 407 additions and 499 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
extra/lib/plausible/stats/funnel.ex
View File

@ -10,7 +10,7 @@ defmodule Plausible.Stats.Funnel do
alias Plausible.Funnels
import Ecto.Query
import Plausible.Stats.Fragments
import Plausible.Stats.SQL.Fragments
alias Plausible.ClickhouseRepo
alias Plausible.Stats.Base

20
extra/lib/plausible/stats/goal/revenue.ex
View File

@ -12,26 +12,6 @@ defmodule Plausible.Stats.Goal.Revenue do
@revenue_metrics
end
def total_revenue_query() do
dynamic(
[e],
selected_as(
fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount),
:total_revenue
)
)
end
def average_revenue_query() do
dynamic(
[e],
selected_as(
fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount),
:average_revenue
)
)
end
@spec get_revenue_tracking_currency(Plausible.Site.t(), Plausible.Stats.Query.t(), [atom()]) ::
{atom() | nil, [atom()]}
@doc """

2
lib/plausible/exports.ex
View File

@ -4,7 +4,7 @@ defmodule Plausible.Exports do
"""
use Plausible
use Plausible.Stats.Fragments
use Plausible.Stats.SQL.Fragments
import Ecto.Query
@doc "Schedules CSV export job to S3 storage"

5
lib/plausible/stats/aggregate.ex
View File

@ -3,7 +3,7 @@ defmodule Plausible.Stats.Aggregate do
use Plausible
import Plausible.Stats.Base
import Ecto.Query
alias Plausible.Stats.{Query, Util}
alias Plausible.Stats.{Query, Util, SQL}
def aggregate(site, query, metrics) do
{currency, metrics} =
@ -64,8 +64,7 @@ defmodule Plausible.Stats.Aggregate do
timed_page_transitions_q =
from e in Ecto.Query.subquery(windowed_pages_q),
group_by: [e.pathname, e.next_pathname, e.session_id],
where:
^Plausible.Stats.Filters.WhereBuilder.build_condition(:pathname, event_page_filter),
where: ^SQL.WhereBuilder.build_condition(:pathname, event_page_filter),
where: e.next_timestamp != 0,
select: %{
pathname: e.pathname,

261
lib/plausible/stats/base.ex
View File

@ -1,14 +1,12 @@
defmodule Plausible.Stats.Base do
use Plausible.ClickhouseRepo
use Plausible
use Plausible.Stats.Fragments
use Plausible.Stats.SQL.Fragments
alias Plausible.Stats.{Query, Filters, TableDecider}
alias Plausible.Stats.{Query, TableDecider, SQL}
alias Plausible.Timezones
import Ecto.Query
@uniq_users_expression "toUInt64(round(uniq(?) * any(_sample_factor)))"
def base_event_query(site, query) do
events_q = query_events(site, query)
@ -32,7 +30,7 @@ defmodule Plausible.Stats.Base do
end
def query_events(site, query) do
q = from(e in "events_v2", where: ^Filters.WhereBuilder.build(:events, site, query))
q = from(e in "events_v2", where: ^SQL.WhereBuilder.build(:events, site, query))
on_ee do
q = Plausible.Stats.Sampling.add_query_hint(q, query)
@ -42,7 +40,7 @@ defmodule Plausible.Stats.Base do
end
def query_sessions(site, query) do
q = from(s in "sessions_v2", where: ^Filters.WhereBuilder.build(:sessions, site, query))
q = from(s in "sessions_v2", where: ^SQL.WhereBuilder.build(:sessions, site, query))
on_ee do
q = Plausible.Stats.Sampling.add_query_hint(q, query)
@ -53,206 +51,16 @@ defmodule Plausible.Stats.Base do
def select_event_metrics(metrics) do
metrics
|> Enum.map(&select_event_metric/1)
|> Enum.map(&SQL.Expression.event_metric/1)
|> Enum.reduce(%{}, &Map.merge/2)
end
defp select_event_metric(:pageviews) do
%{
pageviews:
dynamic(
[e],
selected_as(
fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name),
:pageviews
)
)
}
end
defp select_event_metric(:events) do
%{
events:
dynamic(
[],
selected_as(fragment("toUInt64(round(count(*) * any(_sample_factor)))"), :events)
)
}
end
defp select_event_metric(:visitors) do
%{
visitors: dynamic([e], selected_as(fragment(@uniq_users_expression, e.user_id), :visitors))
}
end
defp select_event_metric(:visits) do
%{
visits:
dynamic(
[e],
selected_as(
fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.session_id),
:visits
)
)
}
end
on_ee do
defp select_event_metric(:total_revenue) do
%{total_revenue: Plausible.Stats.Goal.Revenue.total_revenue_query()}
end
defp select_event_metric(:average_revenue) do
%{average_revenue: Plausible.Stats.Goal.Revenue.average_revenue_query()}
end
end
defp select_event_metric(:sample_percent) do
%{
sample_percent:
dynamic(
[],
fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)")
)
}
end
defp select_event_metric(:percentage), do: %{}
defp select_event_metric(:conversion_rate), do: %{}
defp select_event_metric(:group_conversion_rate), do: %{}
defp select_event_metric(:total_visitors), do: %{}
defp select_event_metric(unknown), do: raise("Unknown metric: #{unknown}")
def select_session_metrics(metrics, query) do
metrics
|> Enum.map(&select_session_metric(&1, query))
|> Enum.map(&SQL.Expression.session_metric(&1, query))
|> Enum.reduce(%{}, &Map.merge/2)
end
defp select_session_metric(:bounce_rate, query) do
# :TRICKY: If page is passed to query, we only count bounce rate where users _entered_ at page.
event_page_filter = Query.get_filter(query, "event:page")
condition = Filters.WhereBuilder.build_condition(:entry_page, event_page_filter)
%{
bounce_rate:
dynamic(
[],
selected_as(
fragment(
"toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))",
^condition,
^condition
),
:bounce_rate
)
),
__internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))
}
end
defp select_session_metric(:visits, _query) do
%{
visits:
dynamic(
[s],
selected_as(
fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign),
:visits
)
)
}
end
defp select_session_metric(:pageviews, _query) do
%{
pageviews:
dynamic(
[s],
selected_as(
fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews),
:pageviews
)
)
}
end
defp select_session_metric(:events, _query) do
%{
events:
dynamic(
[s],
selected_as(
fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events),
:events
)
)
}
end
defp select_session_metric(:visitors, _query) do
%{
visitors:
dynamic(
[s],
selected_as(
fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", s.user_id),
:visitors
)
)
}
end
defp select_session_metric(:visit_duration, _query) do
%{
visit_duration:
dynamic(
[],
selected_as(
fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))"),
:visit_duration
)
),
__internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))
}
end
defp select_session_metric(:views_per_visit, _query) do
%{
views_per_visit:
dynamic(
[s],
selected_as(
fragment(
"ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)",
s.sign,
s.pageviews,
s.sign
),
:views_per_visit
)
),
__internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))
}
end
defp select_session_metric(:sample_percent, _query) do
%{
sample_percent:
dynamic(
[],
fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)")
)
}
end
defp select_session_metric(:percentage, _query), do: %{}
defp select_session_metric(:conversion_rate, _query), do: %{}
defp select_session_metric(:group_conversion_rate, _query), do: %{}
def filter_converted_sessions(db_query, site, query) do
if Query.has_event_filters?(query) do
converted_sessions =
@ -334,7 +142,9 @@ defmodule Plausible.Stats.Base do
defp total_visitors(site, query) do
base_event_query(site, query)
|> select([e], total_visitors: fragment(@uniq_users_expression, e.user_id))
|> select([e],
total_visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.user_id)
)
end
# `total_visitors_subquery` returns a subquery which selects `total_visitors` -
@ -350,18 +160,17 @@ defmodule Plausible.Stats.Base do
def total_visitors_subquery(site, query, include_imported)
def total_visitors_subquery(site, query, true = _include_imported) do
dynamic(
[e],
selected_as(
wrap_alias([], %{
total_visitors:
subquery(total_visitors(site, query)) +
subquery(Plausible.Stats.Imported.total_imported_visitors(site, query)),
:__total_visitors
)
)
subquery(Plausible.Stats.Imported.total_imported_visitors(site, query))
})
end
def total_visitors_subquery(site, query, false = _include_imported) do
dynamic([e], selected_as(subquery(total_visitors(site, query)), :__total_visitors))
wrap_alias([], %{
total_visitors: subquery(total_visitors(site, query))
})
end
def add_percentage_metric(q, site, query, metrics) do
@ -369,19 +178,14 @@ defmodule Plausible.Stats.Base do
total_query = Query.set_dimensions(query, [])
q
|> select_merge(
^%{__total_visitors: total_visitors_subquery(site, total_query, query.include_imported)}
)
|> select_merge(%{
|> select_merge_as([], total_visitors_subquery(site, total_query, query.include_imported))
|> select_merge_as([], %{
percentage:
selected_as(
fragment(
"if(? > 0, round(? / ? * 100, 1), null)",
selected_as(:__total_visitors),
selected_as(:visitors),
selected_as(:__total_visitors)
),
:percentage
fragment(
"if(? > 0, round(? / ? * 100, 1), null)",
selected_as(:total_visitors),
selected_as(:visitors),
selected_as(:total_visitors)
)
})
else
@ -401,19 +205,14 @@ defmodule Plausible.Stats.Base do
# :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALL
subquery(q)
|> select_merge(
^%{total_visitors: total_visitors_subquery(site, total_query, query.include_imported)}
)
|> select_merge([e], %{
|> select_merge_as([], total_visitors_subquery(site, total_query, query.include_imported))
|> select_merge_as([e], %{
conversion_rate:
selected_as(
fragment(
"if(? > 0, round(? / ? * 100, 1), 0)",
selected_as(:__total_visitors),
e.visitors,
selected_as(:__total_visitors)
),
:conversion_rate
fragment(
"if(? > 0, round(? / ? * 100, 1), 0)",
selected_as(:total_visitors),
e.visitors,
selected_as(:total_visitors)
)
})
else

2
lib/plausible/stats/breakdown.ex
View File

@ -1,7 +1,7 @@
defmodule Plausible.Stats.Breakdown do
use Plausible.ClickhouseRepo
use Plausible
use Plausible.Stats.Fragments
use Plausible.Stats.SQL.Fragments
import Plausible.Stats.Base
import Ecto.Query

2
lib/plausible/stats/clickhouse.ex
View File

@ -2,7 +2,7 @@ defmodule Plausible.Stats.Clickhouse do
use Plausible
use Plausible.Repo
use Plausible.ClickhouseRepo
use Plausible.Stats.Fragments
use Plausible.Stats.SQL.Fragments
import Ecto.Query, only: [from: 2]

2
lib/plausible/stats/current_visitors.ex
View File

@ -1,6 +1,6 @@
defmodule Plausible.Stats.CurrentVisitors do
use Plausible.ClickhouseRepo
use Plausible.Stats.Fragments
use Plausible.Stats.SQL.Fragments
def current_visitors(site) do
first_datetime =

2
lib/plausible/stats/filter_suggestions.ex
View File

@ -1,7 +1,7 @@
defmodule Plausible.Stats.FilterSuggestions do
use Plausible.Repo
use Plausible.ClickhouseRepo
use Plausible.Stats.Fragments
use Plausible.Stats.SQL.Fragments
import Plausible.Stats.Base
import Ecto.Query

7
lib/plausible/stats/imported/base.ex
View File

@ -6,8 +6,7 @@ defmodule Plausible.Stats.Imported.Base do
import Ecto.Query
alias Plausible.Imported
alias Plausible.Stats.Filters
alias Plausible.Stats.Query
alias Plausible.Stats.{Filters, Query, SQL}
@property_to_table_mappings %{
"visit:source" => "imported_sources",
@ -213,9 +212,9 @@ defmodule Plausible.Stats.Imported.Base do
defp apply_filter(q, %Query{filters: filters}) do
Enum.reduce(filters, q, fn [_, filter_key | _] = filter, q ->
db_field = Plausible.Stats.Filters.without_prefix(filter_key)
db_field = Filters.without_prefix(filter_key)
mapped_db_field = Map.get(@db_field_mappings, db_field, db_field)
condition = Filters.WhereBuilder.build_condition(mapped_db_field, filter)
condition = SQL.WhereBuilder.build_condition(mapped_db_field, filter)
where(q, ^condition)
end)

137
lib/plausible/stats/imported/imported.ex
View File

@ -1,11 +1,11 @@
defmodule Plausible.Stats.Imported do
alias Plausible.Stats.Filters
use Plausible.ClickhouseRepo
use Plausible.Stats.SQL.Fragments
import Ecto.Query
import Plausible.Stats.Fragments
import Plausible.Stats.Util, only: [shortname: 2]
alias Plausible.Stats.Filters
alias Plausible.Stats.Imported
alias Plausible.Stats.Query
alias Plausible.Stats.SQL.QueryBuilder
@ -290,12 +290,8 @@ defmodule Plausible.Stats.Imported do
"imported_custom_events" ->
Imported.Base.query_imported("imported_custom_events", site, query)
|> where([i], i.visitors > 0)
|> select_merge([i], %{
dim0:
selected_as(
fragment("-indexOf(?, ?)", type(^events, {:array, :string}), i.name),
:dim0
)
|> select_merge_as([i], %{
dim0: fragment("-indexOf(?, ?)", type(^events, {:array, :string}), i.name)
})
|> select_imported_metrics(metrics)
|> group_by([], selected_as(:dim0))
@ -314,8 +310,8 @@ defmodule Plausible.Stats.Imported do
)
|> join(:array, index in fragment("indices"))
|> group_by([_i, index], index)
|> select_merge([_i, index], %{
dim0: selected_as(type(fragment("?", index), :integer), :dim0)
|> select_merge_as([_i, index], %{
dim0: type(fragment("?", index), :integer)
})
|> select_imported_metrics(metrics)
end)
@ -563,17 +559,8 @@ defmodule Plausible.Stats.Imported do
defp group_imported_by(q, dim, key) when dim in [:source, :referrer] do
q
|> group_by([i], field(i, ^dim))
|> select_merge([i], %{
^key =>
selected_as(
fragment(
"if(empty(?), ?, ?)",
field(i, ^dim),
@no_ref,
field(i, ^dim)
),
^key
)
|> select_merge_as([i], %{
key => fragment("if(empty(?), ?, ?)", field(i, ^dim), @no_ref, field(i, ^dim))
})
end
@ -582,90 +569,70 @@ defmodule Plausible.Stats.Imported do
q
|> group_by([i], field(i, ^dim))
|> where([i], fragment("not empty(?)", field(i, ^dim)))
|> select_merge([i], %{^key => selected_as(field(i, ^dim), ^key)})
|> select_merge_as([i], %{key => field(i, ^dim)})
end
defp group_imported_by(q, :page, key) do
q
|> group_by([i], i.page)
|> select_merge([i], %{^key => selected_as(i.page, ^key), time_on_page: sum(i.time_on_page)})
|> select_merge_as([i], %{key => i.page, time_on_page: sum(i.time_on_page)})
end
defp group_imported_by(q, :country, key) do
q
|> group_by([i], i.country)
|> where([i], i.country != "ZZ")
|> select_merge([i], %{^key => selected_as(i.country, ^key)})
|> select_merge_as([i], %{key => i.country})
end
defp group_imported_by(q, :region, key) do
q
|> group_by([i], i.region)
|> where([i], i.region != "")
|> select_merge([i], %{^key => selected_as(i.region, ^key)})
|> select_merge_as([i], %{key => i.region})
end
defp group_imported_by(q, :city, key) do
q
|> group_by([i], i.city)
|> where([i], i.city != 0 and not is_nil(i.city))
|> select_merge([i], %{^key => selected_as(i.city, ^key)})
|> select_merge_as([i], %{key => i.city})
end
defp group_imported_by(q, dim, key) when dim in [:device, :browser] do
q
|> group_by([i], field(i, ^dim))
|> select_merge([i], %{
^key =>
selected_as(
fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim)),
^key
)
|> select_merge_as([i], %{
key => fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim))
})
end
defp group_imported_by(q, :browser_version, key) do
q
|> group_by([i], [i.browser_version])
|> select_merge([i], %{
^key =>
selected_as(
fragment(
"if(empty(?), ?, ?)",
i.browser_version,
@not_set,
i.browser_version
),
^key
)
|> select_merge_as([i], %{
key => fragment("if(empty(?), ?, ?)", i.browser_version, @not_set, i.browser_version)
})
end
defp group_imported_by(q, :os, key) do
q
|> group_by([i], i.operating_system)
|> select_merge([i], %{
^key =>
selected_as(
fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system),
^key
)
|> select_merge_as([i], %{
key => fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system)
})
end
defp group_imported_by(q, :os_version, key) do
q
|> group_by([i], [i.operating_system_version])
|> select_merge([i], %{
^key =>
selected_as(
fragment(
"if(empty(?), ?, ?)",
i.operating_system_version,
@not_set,
i.operating_system_version
),
^key
|> select_merge_as([i], %{
key =>
fragment(
"if(empty(?), ?, ?)",
i.operating_system_version,
@not_set,
i.operating_system_version
)
})
end
@ -673,28 +640,28 @@ defmodule Plausible.Stats.Imported do
defp group_imported_by(q, dim, key) when dim in [:entry_page, :exit_page] do
q
|> group_by([i], field(i, ^dim))
|> select_merge([i], %{^key => selected_as(field(i, ^dim), ^key)})
|> select_merge_as([i], %{key => field(i, ^dim)})
end
defp group_imported_by(q, :name, key) do
q
|> group_by([i], i.name)
|> select_merge([i], %{^key => selected_as(i.name, ^key)})
|> select_merge_as([i], %{key => i.name})
end
defp group_imported_by(q, :url, key) do
q
|> group_by([i], i.link_url)
|> select_merge([i], %{
^key => selected_as(fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none), ^key)
|> select_merge_as([i], %{
key => fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)
})
end
defp group_imported_by(q, :path, key) do
q
|> group_by([i], i.path)
|> select_merge([i], %{
^key => selected_as(fragment("if(not empty(?), ?, ?)", i.path, i.path, @none), ^key)
|> select_merge_as([i], %{
key => fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)
})
end
@ -705,23 +672,14 @@ defmodule Plausible.Stats.Imported do
end
defp select_joined_dimension(q, "visit:city", key) do
select_merge(q, [s, i], %{
^key => selected_as(fragment("greatest(?,?)", field(i, ^key), field(s, ^key)), ^key)
select_merge_as(q, [s, i], %{
key => fragment("greatest(?,?)", field(i, ^key), field(s, ^key))
})
end
defp select_joined_dimension(q, _dimension, key) do
select_merge(q, [s, i], %{
^key =>
selected_as(
fragment(
"if(empty(?), ?, ?)",
field(s, ^key),
field(i, ^key),
field(s, ^key)
),
^key
)
select_merge_as(q, [s, i], %{
key => fragment("if(empty(?), ?, ?)", field(s, ^key), field(i, ^key), field(s, ^key))
})
end
@ -734,31 +692,31 @@ defmodule Plausible.Stats.Imported do
defp select_joined_metrics(q, [:visits | rest]) do
q
|> select_merge([s, i], %{visits: selected_as(s.visits + i.visits, :visits)})
|> select_merge_as([s, i], %{visits: s.visits + i.visits})
|> select_joined_metrics(rest)
end
defp select_joined_metrics(q, [:visitors | rest]) do
q
|> select_merge([s, i], %{visitors: selected_as(s.visitors + i.visitors, :visitors)})
|> select_merge_as([s, i], %{visitors: s.visitors + i.visitors})
|> select_joined_metrics(rest)
end
defp select_joined_metrics(q, [:events | rest]) do
q
|> select_merge([s, i], %{events: selected_as(s.events + i.events, :events)})
|> select_merge_as([s, i], %{events: s.events + i.events})
|> select_joined_metrics(rest)
end
defp select_joined_metrics(q, [:pageviews | rest]) do
q
|> select_merge([s, i], %{pageviews: selected_as(s.pageviews + i.pageviews, :pageviews)})
|> select_merge_as([s, i], %{pageviews: s.pageviews + i.pageviews})
|> select_joined_metrics(rest)
end
defp select_joined_metrics(q, [:views_per_visit | rest]) do
q
|> select_merge([s, i], %{
|> select_merge_as([s, i], %{
views_per_visit:
fragment(
"if(? + ? > 0, round((? + ? * ?) / (? + ?), 2), 0)",
@ -776,7 +734,7 @@ defmodule Plausible.Stats.Imported do
defp select_joined_metrics(q, [:bounce_rate | rest]) do
q
|> select_merge([s, i], %{
|> select_merge_as([s, i], %{
bounce_rate:
fragment(
"if(? + ? > 0, round(100 * (? + (? * ? / 100)) / (? + ?)), 0)",
@ -794,7 +752,7 @@ defmodule Plausible.Stats.Imported do
defp select_joined_metrics(q, [:visit_duration | rest]) do
q
|> select_merge([s, i], %{
|> select_merge_as([s, i], %{
visit_duration:
fragment(
"""
@ -818,7 +776,7 @@ defmodule Plausible.Stats.Imported do
defp select_joined_metrics(q, [:sample_percent | rest]) do
q
|> select_merge([s, i], %{sample_percent: s.sample_percent})
|> select_merge_as([s, i], %{sample_percent: s.sample_percent})
|> select_joined_metrics(rest)
end
@ -831,10 +789,11 @@ defmodule Plausible.Stats.Imported do
from(a in subquery(q1),
full_join: b in subquery(q2),
on: a.dim0 == b.dim0,
select: %{
dim0: selected_as(fragment("if(? != 0, ?, ?)", a.dim0, a.dim0, b.dim0), :dim0)
}
select: %{}
)
|> select_merge_as([a, b], %{
dim0: fragment("if(? != 0, ?, ?)", a.dim0, a.dim0, b.dim0)
})
|> select_joined_metrics(metrics)
end
end

286
lib/plausible/stats/sql/expression.ex
View File

@ -1,131 +1,253 @@
defmodule Plausible.Stats.SQL.Expression do
@moduledoc """
This module is responsible for generating SQL/Ecto expressions
for dimensions used in query select, group_by and order_by.
for dimensions and metrics used in query SELECT statement.
Each dimension and metric is tagged with with selected_as for easier
usage down the line.
"""
use Plausible
use Plausible.Stats.SQL.Fragments
import Ecto.Query
use Plausible.Stats.Fragments
alias Plausible.Stats.{Query, SQL}
@no_ref "Direct / None"
@not_set "(not set)"
defmacrop field_or_blank_value(expr, empty_value, select_alias) do
defmacrop field_or_blank_value(key, expr, empty_value) do
quote do
dynamic(
[t],
selected_as(
fragment("if(empty(?), ?, ?)", unquote(expr), unquote(empty_value), unquote(expr)),
^unquote(select_alias)
)
)
wrap_alias([t], %{
unquote(key) =>
fragment("if(empty(?), ?, ?)", unquote(expr), unquote(empty_value), unquote(expr))
})
end
end
def dimension("time:hour", query, select_alias) do
dynamic(
[t],
selected_as(
fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone),
^select_alias
)
)
def dimension(key, "time:hour", query) do
wrap_alias([t], %{
key => fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone)
})
end
def dimension("time:day", query, select_alias) do
dynamic(
[t],
selected_as(
fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone),
^select_alias
)
)
def dimension(key, "time:day", query) do
wrap_alias([t], %{
key => fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone)
})
end
def dimension("time:month", query, select_alias) do
dynamic(
[t],
selected_as(
fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone),
^select_alias
)
)
def dimension(key, "time:month", query) do
wrap_alias([t], %{
key => fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone)
})
end
def dimension("event:name", _query, select_alias),
do: dynamic([t], selected_as(t.name, ^select_alias))
def dimension(key, "event:name", _query),
do: wrap_alias([t], %{key => t.name})
def dimension("event:page", _query, select_alias),
do: dynamic([t], selected_as(t.pathname, ^select_alias))
def dimension(key, "event:page", _query),
do: wrap_alias([t], %{key => t.pathname})
def dimension("event:hostname", _query, select_alias),
do: dynamic([t], selected_as(t.hostname, ^select_alias))
def dimension(key, "event:hostname", _query),
do: wrap_alias([t], %{key => t.hostname})
def dimension("event:props:" <> property_name, _query, select_alias) do
dynamic(
[t],
selected_as(
def dimension(key, "event:props:" <> property_name, _query) do
wrap_alias([t], %{
key =>
fragment(
"if(not empty(?), ?, '(none)')",
get_by_key(t, :meta, ^property_name),
get_by_key(t, :meta, ^property_name)
),
^select_alias
)
)
)
})
end
def dimension("visit:entry_page", _query, select_alias),
do: dynamic([t], selected_as(t.entry_page, ^select_alias))
def dimension(key, "visit:entry_page", _query),
do: wrap_alias([t], %{key => t.entry_page})
def dimension("visit:exit_page", _query, select_alias),
do: dynamic([t], selected_as(t.exit_page, ^select_alias))
def dimension(key, "visit:exit_page", _query),
do: wrap_alias([t], %{key => t.exit_page})
def dimension("visit:utm_medium", _query, select_alias),
do: field_or_blank_value(t.utm_medium, @not_set, select_alias)
def dimension(key, "visit:utm_medium", _query),
do: field_or_blank_value(key, t.utm_medium, @not_set)
def dimension("visit:utm_source", _query, select_alias),
do: field_or_blank_value(t.utm_source, @not_set, select_alias)
def dimension(key, "visit:utm_source", _query),
do: field_or_blank_value(key, t.utm_source, @not_set)
def dimension("visit:utm_campaign", _query, select_alias),
do: field_or_blank_value(t.utm_campaign, @not_set, select_alias)
def dimension(key, "visit:utm_campaign", _query),
do: field_or_blank_value(key, t.utm_campaign, @not_set)
def dimension("visit:utm_content", _query, select_alias),
do: field_or_blank_value(t.utm_content, @not_set, select_alias)
def dimension(key, "visit:utm_content", _query),
do: field_or_blank_value(key, t.utm_content, @not_set)
def dimension("visit:utm_term", _query, select_alias),
do: field_or_blank_value(t.utm_term, @not_set, select_alias)
def dimension(key, "visit:utm_term", _query),
do: field_or_blank_value(key, t.utm_term, @not_set)
def dimension("visit:source", _query, select_alias),
do: field_or_blank_value(t.source, @no_ref, select_alias)
def dimension(key, "visit:source", _query),
do: field_or_blank_value(key, t.source, @no_ref)
def dimension("visit:referrer", _query, select_alias),
do: field_or_blank_value(t.referrer, @no_ref, select_alias)
def dimension(key, "visit:referrer", _query),
do: field_or_blank_value(key, t.referrer, @no_ref)
def dimension("visit:device", _query, select_alias),
do: field_or_blank_value(t.device, @not_set, select_alias)
def dimension(key, "visit:device", _query),
do: field_or_blank_value(key, t.device, @not_set)
def dimension("visit:os", _query, select_alias),
do: field_or_blank_value(t.os, @not_set, select_alias)
def dimension(key, "visit:os", _query),
do: field_or_blank_value(key, t.os, @not_set)
def dimension("visit:os_version", _query, select_alias),
do: field_or_blank_value(t.os_version, @not_set, select_alias)
def dimension(key, "visit:os_version", _query),
do: field_or_blank_value(key, t.os_version, @not_set)
def dimension("visit:browser", _query, select_alias),
do: field_or_blank_value(t.browser, @not_set, select_alias)
def dimension(key, "visit:browser", _query),
do: field_or_blank_value(key, t.browser, @not_set)
def dimension("visit:browser_version", _query, select_alias),
do: field_or_blank_value(t.browser_version, @not_set, select_alias)
def dimension(key, "visit:browser_version", _query),
do: field_or_blank_value(key, t.browser_version, @not_set)
def dimension("visit:country", _query, select_alias),
do: dynamic([t], selected_as(t.country, ^select_alias))
def dimension(key, "visit:country", _query),
do: wrap_alias([t], %{key => t.country})
def dimension("visit:region", _query, select_alias),
do: dynamic([t], selected_as(t.region, ^select_alias))
def dimension(key, "visit:region", _query),
do: wrap_alias([t], %{key => t.region})
def dimension("visit:city", _query, select_alias),
do: dynamic([t], selected_as(t.city, ^select_alias))
def dimension(key, "visit:city", _query),
do: wrap_alias([t], %{key => t.city})
def event_metric(:pageviews) do
wrap_alias([e], %{
pageviews:
fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name)
})
end
def event_metric(:events) do
wrap_alias([], %{
events: fragment("toUInt64(round(count(*) * any(_sample_factor)))")
})
end
def event_metric(:visitors) do
wrap_alias([e], %{
visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.user_id)
})
end
def event_metric(:visits) do
wrap_alias([e], %{
visits: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.session_id)
})
end
on_ee do
def event_metric(:total_revenue) do
wrap_alias(
[e],
%{
total_revenue:
fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)
}
)
end
def event_metric(:average_revenue) do
wrap_alias(
[e],
%{
average_revenue:
fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)
}
)
end
end
def event_metric(:sample_percent) do
wrap_alias([], %{
sample_percent:
fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)")
})
end
def event_metric(:percentage), do: %{}
def event_metric(:conversion_rate), do: %{}
def event_metric(:group_conversion_rate), do: %{}
def event_metric(:total_visitors), do: %{}
def event_metric(unknown), do: raise("Unknown metric: #{unknown}")
def session_metric(:bounce_rate, query) do
# :TRICKY: If page is passed to query, we only count bounce rate where users _entered_ at page.
event_page_filter = Query.get_filter(query, "event:page")
condition = SQL.WhereBuilder.build_condition(:entry_page, event_page_filter)
wrap_alias([], %{
bounce_rate:
fragment(
"toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))",
^condition,
^condition
),
__internal_visits: fragment("toUInt32(sum(sign))")
})
end
def session_metric(:visits, _query) do
wrap_alias([s], %{
visits: fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign)
})
end
def session_metric(:pageviews, _query) do
wrap_alias([s], %{
pageviews:
fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews)
})
end
def session_metric(:events, _query) do
wrap_alias([s], %{
events: fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events)
})
end
def session_metric(:visitors, _query) do
wrap_alias([s], %{
visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", s.user_id)
})
end
def session_metric(:visit_duration, _query) do
wrap_alias([], %{
visit_duration:
fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))"),
__internal_visits: fragment("toUInt32(sum(sign))")
})
end
def session_metric(:views_per_visit, _query) do
wrap_alias([s], %{
views_per_visit:
fragment(
"ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)",
s.sign,
s.pageviews,
s.sign
),
__internal_visits: fragment("toUInt32(sum(sign))")
})
end
def session_metric(:sample_percent, _query) do
wrap_alias([], %{
sample_percent:
fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)")
})
end
def session_metric(:percentage, _query), do: %{}
def session_metric(:conversion_rate, _query), do: %{}
def session_metric(:group_conversion_rate, _query), do: %{}
defmacro event_goal_join(events, page_regexes) do
quote do

79
lib/plausible/stats/fragments.ex → lib/plausible/stats/sql/fragments.ex
View File

@ -1,4 +1,15 @@
defmodule Plausible.Stats.Fragments do
defmodule Plausible.Stats.SQL.Fragments do
@moduledoc """
Various macros and common SQL fragments used in Stats code.
"""
defmacro __using__(_) do
quote do
import Plausible.Stats.SQL.Fragments
require Plausible.Stats.SQL.Fragments
end
end
defmacro uniq(user_id) do
quote do
fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", unquote(user_id))
@ -56,21 +67,23 @@ defmodule Plausible.Stats.Fragments do
`not_before` boundary is set to the past Saturday, which is before the
weekstart, therefore the cap does not apply.
iex> this_wednesday = ~D[2022-11-09]
...> past_saturday = ~D[2022-11-05]
...> weekstart_not_before(this_wednesday, past_saturday)
```
> this_wednesday = ~D[2022-11-09]
> past_saturday = ~D[2022-11-05]
> weekstart_not_before(this_wednesday, past_saturday)
~D[2022-11-07]
```
In this other example, the fragment returns Tuesday and not the weekstart.
The `not_before` boundary is set to Tuesday, which is past the weekstart,
therefore the cap applies.
iex> this_wednesday = ~D[2022-11-09]
...> this_tuesday = ~D[2022-11-08]
...> weekstart_not_before(this_wednesday, this_tuesday)
```
> this_wednesday = ~D[2022-11-09]
> this_tuesday = ~D[2022-11-08]
> weekstart_not_before(this_wednesday, this_tuesday)
~D[2022-11-08]
```
"""
defmacro weekstart_not_before(date, not_before) do
quote do
@ -85,7 +98,7 @@ defmodule Plausible.Stats.Fragments do
end
@doc """
Same as Plausible.Stats.Fragments.weekstart_not_before/2 but converts dates to
Same as Plausible.Stats.SQL.Fragments.weekstart_not_before/2 but converts dates to
the specified timezone.
"""
defmacro weekstart_not_before(date, not_before, timezone) do
@ -143,9 +156,51 @@ defmodule Plausible.Stats.Fragments do
def meta_value_column(:meta), do: :"meta.value"
def meta_value_column(:entry_meta), do: :"entry_meta.value"
defmacro __using__(_) do
@doc """
Convenience Ecto macro for wrapping a map passed to select_merge_as such that each
expression gets wrapped in dynamic and set as selected_as.
### Examples
iex> wrap_alias([t], %{ foo: t.column }) |> expand_macro_once
"%{foo: dynamic([t], selected_as(t.column, :foo))}"
"""
defmacro wrap_alias(binding, map_literal) do
update_literal_map_values(map_literal, fn {key, expr} ->
key_expr =
if Macro.quoted_literal?(key) do
key
else
quote(do: ^unquote(key))
end
quote(do: dynamic(unquote(binding), selected_as(unquote(expr), unquote(key_expr))))
end)
end
@doc """
Convenience Ecto macro for wrapping select_merge where each value gets in turn passed to selected_as.
### Examples
iex> select_merge_as(q, [t], %{ foo: t.column }) |> expand_macro_once
"select_merge(q, [], ^wrap_alias([t], %{foo: t.column}))"
"""
defmacro select_merge_as(q, binding, map_literal) do
quote do
import Plausible.Stats.Fragments
select_merge(unquote(q), [], ^wrap_alias(unquote(binding), unquote(map_literal)))
end
end
defp update_literal_map_values({:%{}, ctx, keyword_list}, mapper_fn) do
{
:%{},
ctx,
Enum.map(keyword_list, fn {key, expr} ->
{key, mapper_fn.({key, expr})}
end)
}
end
defp update_literal_map_values(ast, _), do: ast
end

82
lib/plausible/stats/sql/query_builder.ex
View File

@ -2,12 +2,13 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
@moduledoc false
use Plausible
use Plausible.Stats.SQL.Fragments
import Ecto.Query
import Plausible.Stats.Imported
import Plausible.Stats.Util
alias Plausible.Stats.{Base, Query, QueryOptimizer, TableDecider, Filters}
alias Plausible.Stats.{Base, Filters, Query, QueryOptimizer, TableDecider, SQL}
alias Plausible.Stats.SQL.Expression
require Plausible.Stats.SQL.Expression
@ -30,7 +31,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
q =
from(
e in "events_v2",
where: ^Filters.WhereBuilder.build(:events, site, events_query),
where: ^SQL.WhereBuilder.build(:events, site, events_query),
select: ^Base.select_event_metrics(events_query.metrics)
)
@ -73,7 +74,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
q =
from(
e in "sessions_v2",
where: ^Filters.WhereBuilder.build(:sessions, site, sessions_query),
where: ^SQL.WhereBuilder.build(:sessions, site, sessions_query),
select: ^Base.select_session_metrics(sessions_query.metrics, sessions_query)
)
@ -94,7 +95,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
if Query.has_event_filters?(query) do
events_q =
from(e in "events_v2",
where: ^Filters.WhereBuilder.build(:events, site, query),
where: ^SQL.WhereBuilder.build(:events, site, query),
select: %{
session_id: fragment("DISTINCT ?", e.session_id),
_sample_factor: fragment("_sample_factor")
@ -135,7 +136,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
key = shortname(query, dimension)
q
|> select_merge(^%{key => Expression.dimension(dimension, query, key)})
|> select_merge_as([], Expression.dimension(key, dimension, query))
|> group_by([], selected_as(^key))
end
@ -147,9 +148,9 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
order_by(
q,
[t],
^{
order_direction,
dynamic([], selected_as(^shortname(query, metric_or_dimension)))
{
^order_direction,
selected_as(^shortname(query, metric_or_dimension))
}
)
end
@ -157,19 +158,11 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
defmacrop select_join_fields(q, query, list, table_name) do
quote do
Enum.reduce(unquote(list), unquote(q), fn metric_or_dimension, q ->
select_merge(
q,
^%{
shortname(unquote(query), metric_or_dimension) =>
dynamic(
[e, s],
selected_as(
field(unquote(table_name), ^shortname(unquote(query), metric_or_dimension)),
^shortname(unquote(query), metric_or_dimension)
)
)
}
)
key = shortname(unquote(query), metric_or_dimension)
select_merge_as(q, [e, s], %{
key => field(unquote(table_name), ^key)
})
end)
end
end
@ -185,21 +178,17 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
|> Query.set_dimensions([])
q
|> select_merge(
^%{
total_visitors: Base.total_visitors_subquery(site, total_query, query.include_imported)
}
|> select_merge_as(
[],
Base.total_visitors_subquery(site, total_query, query.include_imported)
)
|> select_merge([e], %{
|> select_merge_as([e], %{
conversion_rate:
selected_as(
fragment(
"if(? > 0, round(? / ? * 100, 1), 0)",
selected_as(:__total_visitors),
selected_as(:visitors),
selected_as(:__total_visitors)
),
:conversion_rate
fragment(
"if(? > 0, round(? / ? * 100, 1), 0)",
selected_as(:total_visitors),
selected_as(:visitors),
selected_as(:total_visitors)
)
})
else
@ -228,21 +217,18 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
from(e in subquery(q),
left_join: c in subquery(build(group_totals_query, site)),
on: ^build_group_by_join(query),
select_merge: %{
total_visitors: c.visitors,
group_conversion_rate:
selected_as(
fragment(
"if(? > 0, round(? / ? * 100, 1), 0)",
c.visitors,
e.visitors,
c.visitors
),
:group_conversion_rate
)
}
on: ^build_group_by_join(query)
)
|> select_merge_as([e, c], %{
total_visitors: c.visitors,
group_conversion_rate:
fragment(
"if(? > 0, round(? / ? * 100, 1), 0)",
c.visitors,
e.visitors,
c.visitors
)
})
|> select_join_fields(query, query.dimensions, e)
|> select_join_fields(query, List.delete(query.metrics, :group_conversion_rate), e)
else

4
lib/plausible/stats/filters/where_builder.ex → lib/plausible/stats/sql/where_builder.ex
View File

@ -1,4 +1,4 @@
defmodule Plausible.Stats.Filters.WhereBuilder do
defmodule Plausible.Stats.SQL.WhereBuilder do
@moduledoc """
A module for building am ecto where clause of a query out of a query.
"""
@ -8,7 +8,7 @@ defmodule Plausible.Stats.Filters.WhereBuilder do
alias Plausible.Stats.Query
use Plausible.Stats.Fragments
use Plausible.Stats.SQL.Fragments
require Logger

2
lib/plausible/stats/timeseries.ex
View File

@ -4,7 +4,7 @@ defmodule Plausible.Stats.Timeseries do
alias Plausible.Stats.{Query, Util, Imported}
import Plausible.Stats.{Base}
import Ecto.Query
use Plausible.Stats.Fragments
use Plausible.Stats.SQL.Fragments
@typep metric ::
:pageviews

1
lib/plausible/stats/util.ex
View File

@ -6,7 +6,6 @@ defmodule Plausible.Stats.Util do
@manually_removable_metrics [
:__internal_visits,
:visitors,
:__total_visitors,
:__breakdown_value,
:total_visitors
]

10
test/plausible/stats/sql/fragments_test.exs Normal file
View File

@ -0,0 +1,10 @@
defmodule Plausible.Stats.SQL.FragmentsTest do
use ExUnit.Case, async: true
use Plausible.Stats.SQL.Fragments
defmacro expand_macro_once(ast) do
ast |> Macro.expand_once(__ENV__) |> Macro.to_string()
end
doctest Plausible.Stats.SQL.Fragments
end