diff --git a/extra/lib/plausible/stats/funnel.ex b/extra/lib/plausible/stats/funnel.ex index d9c497cee..121a699fa 100644 --- a/extra/lib/plausible/stats/funnel.ex +++ b/extra/lib/plausible/stats/funnel.ex @@ -10,7 +10,7 @@ defmodule Plausible.Stats.Funnel do alias Plausible.Funnels import Ecto.Query - import Plausible.Stats.Fragments + import Plausible.Stats.SQL.Fragments alias Plausible.ClickhouseRepo alias Plausible.Stats.Base diff --git a/extra/lib/plausible/stats/goal/revenue.ex b/extra/lib/plausible/stats/goal/revenue.ex index 7443a1758..1ba564e3f 100644 --- a/extra/lib/plausible/stats/goal/revenue.ex +++ b/extra/lib/plausible/stats/goal/revenue.ex @@ -12,26 +12,6 @@ defmodule Plausible.Stats.Goal.Revenue do @revenue_metrics end - def total_revenue_query() do - dynamic( - [e], - selected_as( - fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount), - :total_revenue - ) - ) - end - - def average_revenue_query() do - dynamic( - [e], - selected_as( - fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount), - :average_revenue - ) - ) - end - @spec get_revenue_tracking_currency(Plausible.Site.t(), Plausible.Stats.Query.t(), [atom()]) :: {atom() | nil, [atom()]} @doc """ diff --git a/lib/plausible/exports.ex b/lib/plausible/exports.ex index d5cb20ada..04e6a4202 100644 --- a/lib/plausible/exports.ex +++ b/lib/plausible/exports.ex @@ -4,7 +4,7 @@ defmodule Plausible.Exports do """ use Plausible - use Plausible.Stats.Fragments + use Plausible.Stats.SQL.Fragments import Ecto.Query @doc "Schedules CSV export job to S3 storage" diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex index 9a2e65bab..8c5cdc4f7 100644 --- a/lib/plausible/stats/aggregate.ex +++ b/lib/plausible/stats/aggregate.ex @@ -3,7 +3,7 @@ defmodule Plausible.Stats.Aggregate do use Plausible import Plausible.Stats.Base import Ecto.Query - alias Plausible.Stats.{Query, Util} + alias Plausible.Stats.{Query, Util, SQL} def aggregate(site, query, metrics) do {currency, metrics} = @@ -64,8 +64,7 @@ defmodule Plausible.Stats.Aggregate do timed_page_transitions_q = from e in Ecto.Query.subquery(windowed_pages_q), group_by: [e.pathname, e.next_pathname, e.session_id], - where: - ^Plausible.Stats.Filters.WhereBuilder.build_condition(:pathname, event_page_filter), + where: ^SQL.WhereBuilder.build_condition(:pathname, event_page_filter), where: e.next_timestamp != 0, select: %{ pathname: e.pathname, diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 5354833a1..8335308cc 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -1,14 +1,12 @@ defmodule Plausible.Stats.Base do use Plausible.ClickhouseRepo use Plausible - use Plausible.Stats.Fragments + use Plausible.Stats.SQL.Fragments - alias Plausible.Stats.{Query, Filters, TableDecider} + alias Plausible.Stats.{Query, TableDecider, SQL} alias Plausible.Timezones import Ecto.Query - @uniq_users_expression "toUInt64(round(uniq(?) * any(_sample_factor)))" - def base_event_query(site, query) do events_q = query_events(site, query) @@ -32,7 +30,7 @@ defmodule Plausible.Stats.Base do end def query_events(site, query) do - q = from(e in "events_v2", where: ^Filters.WhereBuilder.build(:events, site, query)) + q = from(e in "events_v2", where: ^SQL.WhereBuilder.build(:events, site, query)) on_ee do q = Plausible.Stats.Sampling.add_query_hint(q, query) @@ -42,7 +40,7 @@ defmodule Plausible.Stats.Base do end def query_sessions(site, query) do - q = from(s in "sessions_v2", where: ^Filters.WhereBuilder.build(:sessions, site, query)) + q = from(s in "sessions_v2", where: ^SQL.WhereBuilder.build(:sessions, site, query)) on_ee do q = Plausible.Stats.Sampling.add_query_hint(q, query) @@ -53,206 +51,16 @@ defmodule Plausible.Stats.Base do def select_event_metrics(metrics) do metrics - |> Enum.map(&select_event_metric/1) + |> Enum.map(&SQL.Expression.event_metric/1) |> Enum.reduce(%{}, &Map.merge/2) end - defp select_event_metric(:pageviews) do - %{ - pageviews: - dynamic( - [e], - selected_as( - fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name), - :pageviews - ) - ) - } - end - - defp select_event_metric(:events) do - %{ - events: - dynamic( - [], - selected_as(fragment("toUInt64(round(count(*) * any(_sample_factor)))"), :events) - ) - } - end - - defp select_event_metric(:visitors) do - %{ - visitors: dynamic([e], selected_as(fragment(@uniq_users_expression, e.user_id), :visitors)) - } - end - - defp select_event_metric(:visits) do - %{ - visits: - dynamic( - [e], - selected_as( - fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.session_id), - :visits - ) - ) - } - end - - on_ee do - defp select_event_metric(:total_revenue) do - %{total_revenue: Plausible.Stats.Goal.Revenue.total_revenue_query()} - end - - defp select_event_metric(:average_revenue) do - %{average_revenue: Plausible.Stats.Goal.Revenue.average_revenue_query()} - end - end - - defp select_event_metric(:sample_percent) do - %{ - sample_percent: - dynamic( - [], - fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)") - ) - } - end - - defp select_event_metric(:percentage), do: %{} - defp select_event_metric(:conversion_rate), do: %{} - defp select_event_metric(:group_conversion_rate), do: %{} - defp select_event_metric(:total_visitors), do: %{} - - defp select_event_metric(unknown), do: raise("Unknown metric: #{unknown}") - def select_session_metrics(metrics, query) do metrics - |> Enum.map(&select_session_metric(&1, query)) + |> Enum.map(&SQL.Expression.session_metric(&1, query)) |> Enum.reduce(%{}, &Map.merge/2) end - defp select_session_metric(:bounce_rate, query) do - # :TRICKY: If page is passed to query, we only count bounce rate where users _entered_ at page. - event_page_filter = Query.get_filter(query, "event:page") - condition = Filters.WhereBuilder.build_condition(:entry_page, event_page_filter) - - %{ - bounce_rate: - dynamic( - [], - selected_as( - fragment( - "toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))", - ^condition, - ^condition - ), - :bounce_rate - ) - ), - __internal_visits: dynamic([], fragment("toUInt32(sum(sign))")) - } - end - - defp select_session_metric(:visits, _query) do - %{ - visits: - dynamic( - [s], - selected_as( - fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign), - :visits - ) - ) - } - end - - defp select_session_metric(:pageviews, _query) do - %{ - pageviews: - dynamic( - [s], - selected_as( - fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews), - :pageviews - ) - ) - } - end - - defp select_session_metric(:events, _query) do - %{ - events: - dynamic( - [s], - selected_as( - fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events), - :events - ) - ) - } - end - - defp select_session_metric(:visitors, _query) do - %{ - visitors: - dynamic( - [s], - selected_as( - fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", s.user_id), - :visitors - ) - ) - } - end - - defp select_session_metric(:visit_duration, _query) do - %{ - visit_duration: - dynamic( - [], - selected_as( - fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))"), - :visit_duration - ) - ), - __internal_visits: dynamic([], fragment("toUInt32(sum(sign))")) - } - end - - defp select_session_metric(:views_per_visit, _query) do - %{ - views_per_visit: - dynamic( - [s], - selected_as( - fragment( - "ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)", - s.sign, - s.pageviews, - s.sign - ), - :views_per_visit - ) - ), - __internal_visits: dynamic([], fragment("toUInt32(sum(sign))")) - } - end - - defp select_session_metric(:sample_percent, _query) do - %{ - sample_percent: - dynamic( - [], - fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)") - ) - } - end - - defp select_session_metric(:percentage, _query), do: %{} - defp select_session_metric(:conversion_rate, _query), do: %{} - defp select_session_metric(:group_conversion_rate, _query), do: %{} - def filter_converted_sessions(db_query, site, query) do if Query.has_event_filters?(query) do converted_sessions = @@ -334,7 +142,9 @@ defmodule Plausible.Stats.Base do defp total_visitors(site, query) do base_event_query(site, query) - |> select([e], total_visitors: fragment(@uniq_users_expression, e.user_id)) + |> select([e], + total_visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.user_id) + ) end # `total_visitors_subquery` returns a subquery which selects `total_visitors` - @@ -350,18 +160,17 @@ defmodule Plausible.Stats.Base do def total_visitors_subquery(site, query, include_imported) def total_visitors_subquery(site, query, true = _include_imported) do - dynamic( - [e], - selected_as( + wrap_alias([], %{ + total_visitors: subquery(total_visitors(site, query)) + - subquery(Plausible.Stats.Imported.total_imported_visitors(site, query)), - :__total_visitors - ) - ) + subquery(Plausible.Stats.Imported.total_imported_visitors(site, query)) + }) end def total_visitors_subquery(site, query, false = _include_imported) do - dynamic([e], selected_as(subquery(total_visitors(site, query)), :__total_visitors)) + wrap_alias([], %{ + total_visitors: subquery(total_visitors(site, query)) + }) end def add_percentage_metric(q, site, query, metrics) do @@ -369,19 +178,14 @@ defmodule Plausible.Stats.Base do total_query = Query.set_dimensions(query, []) q - |> select_merge( - ^%{__total_visitors: total_visitors_subquery(site, total_query, query.include_imported)} - ) - |> select_merge(%{ + |> select_merge_as([], total_visitors_subquery(site, total_query, query.include_imported)) + |> select_merge_as([], %{ percentage: - selected_as( - fragment( - "if(? > 0, round(? / ? * 100, 1), null)", - selected_as(:__total_visitors), - selected_as(:visitors), - selected_as(:__total_visitors) - ), - :percentage + fragment( + "if(? > 0, round(? / ? * 100, 1), null)", + selected_as(:total_visitors), + selected_as(:visitors), + selected_as(:total_visitors) ) }) else @@ -401,19 +205,14 @@ defmodule Plausible.Stats.Base do # :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALL subquery(q) - |> select_merge( - ^%{total_visitors: total_visitors_subquery(site, total_query, query.include_imported)} - ) - |> select_merge([e], %{ + |> select_merge_as([], total_visitors_subquery(site, total_query, query.include_imported)) + |> select_merge_as([e], %{ conversion_rate: - selected_as( - fragment( - "if(? > 0, round(? / ? * 100, 1), 0)", - selected_as(:__total_visitors), - e.visitors, - selected_as(:__total_visitors) - ), - :conversion_rate + fragment( + "if(? > 0, round(? / ? * 100, 1), 0)", + selected_as(:total_visitors), + e.visitors, + selected_as(:total_visitors) ) }) else diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index a0bdaa150..abf78beac 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -1,7 +1,7 @@ defmodule Plausible.Stats.Breakdown do use Plausible.ClickhouseRepo use Plausible - use Plausible.Stats.Fragments + use Plausible.Stats.SQL.Fragments import Plausible.Stats.Base import Ecto.Query diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex index ed112b9a9..d22a4bb0c 100644 --- a/lib/plausible/stats/clickhouse.ex +++ b/lib/plausible/stats/clickhouse.ex @@ -2,7 +2,7 @@ defmodule Plausible.Stats.Clickhouse do use Plausible use Plausible.Repo use Plausible.ClickhouseRepo - use Plausible.Stats.Fragments + use Plausible.Stats.SQL.Fragments import Ecto.Query, only: [from: 2] diff --git a/lib/plausible/stats/current_visitors.ex b/lib/plausible/stats/current_visitors.ex index 4af146691..3249e510d 100644 --- a/lib/plausible/stats/current_visitors.ex +++ b/lib/plausible/stats/current_visitors.ex @@ -1,6 +1,6 @@ defmodule Plausible.Stats.CurrentVisitors do use Plausible.ClickhouseRepo - use Plausible.Stats.Fragments + use Plausible.Stats.SQL.Fragments def current_visitors(site) do first_datetime = diff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.ex index 8713919e3..c4c59a373 100644 --- a/lib/plausible/stats/filter_suggestions.ex +++ b/lib/plausible/stats/filter_suggestions.ex @@ -1,7 +1,7 @@ defmodule Plausible.Stats.FilterSuggestions do use Plausible.Repo use Plausible.ClickhouseRepo - use Plausible.Stats.Fragments + use Plausible.Stats.SQL.Fragments import Plausible.Stats.Base import Ecto.Query diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.ex index 5b774d9ea..72a888d18 100644 --- a/lib/plausible/stats/imported/base.ex +++ b/lib/plausible/stats/imported/base.ex @@ -6,8 +6,7 @@ defmodule Plausible.Stats.Imported.Base do import Ecto.Query alias Plausible.Imported - alias Plausible.Stats.Filters - alias Plausible.Stats.Query + alias Plausible.Stats.{Filters, Query, SQL} @property_to_table_mappings %{ "visit:source" => "imported_sources", @@ -213,9 +212,9 @@ defmodule Plausible.Stats.Imported.Base do defp apply_filter(q, %Query{filters: filters}) do Enum.reduce(filters, q, fn [_, filter_key | _] = filter, q -> - db_field = Plausible.Stats.Filters.without_prefix(filter_key) + db_field = Filters.without_prefix(filter_key) mapped_db_field = Map.get(@db_field_mappings, db_field, db_field) - condition = Filters.WhereBuilder.build_condition(mapped_db_field, filter) + condition = SQL.WhereBuilder.build_condition(mapped_db_field, filter) where(q, ^condition) end) diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.ex index 57cd6737b..4fcdd04a6 100644 --- a/lib/plausible/stats/imported/imported.ex +++ b/lib/plausible/stats/imported/imported.ex @@ -1,11 +1,11 @@ defmodule Plausible.Stats.Imported do - alias Plausible.Stats.Filters use Plausible.ClickhouseRepo + use Plausible.Stats.SQL.Fragments import Ecto.Query - import Plausible.Stats.Fragments import Plausible.Stats.Util, only: [shortname: 2] + alias Plausible.Stats.Filters alias Plausible.Stats.Imported alias Plausible.Stats.Query alias Plausible.Stats.SQL.QueryBuilder @@ -290,12 +290,8 @@ defmodule Plausible.Stats.Imported do "imported_custom_events" -> Imported.Base.query_imported("imported_custom_events", site, query) |> where([i], i.visitors > 0) - |> select_merge([i], %{ - dim0: - selected_as( - fragment("-indexOf(?, ?)", type(^events, {:array, :string}), i.name), - :dim0 - ) + |> select_merge_as([i], %{ + dim0: fragment("-indexOf(?, ?)", type(^events, {:array, :string}), i.name) }) |> select_imported_metrics(metrics) |> group_by([], selected_as(:dim0)) @@ -314,8 +310,8 @@ defmodule Plausible.Stats.Imported do ) |> join(:array, index in fragment("indices")) |> group_by([_i, index], index) - |> select_merge([_i, index], %{ - dim0: selected_as(type(fragment("?", index), :integer), :dim0) + |> select_merge_as([_i, index], %{ + dim0: type(fragment("?", index), :integer) }) |> select_imported_metrics(metrics) end) @@ -563,17 +559,8 @@ defmodule Plausible.Stats.Imported do defp group_imported_by(q, dim, key) when dim in [:source, :referrer] do q |> group_by([i], field(i, ^dim)) - |> select_merge([i], %{ - ^key => - selected_as( - fragment( - "if(empty(?), ?, ?)", - field(i, ^dim), - @no_ref, - field(i, ^dim) - ), - ^key - ) + |> select_merge_as([i], %{ + key => fragment("if(empty(?), ?, ?)", field(i, ^dim), @no_ref, field(i, ^dim)) }) end @@ -582,90 +569,70 @@ defmodule Plausible.Stats.Imported do q |> group_by([i], field(i, ^dim)) |> where([i], fragment("not empty(?)", field(i, ^dim))) - |> select_merge([i], %{^key => selected_as(field(i, ^dim), ^key)}) + |> select_merge_as([i], %{key => field(i, ^dim)}) end defp group_imported_by(q, :page, key) do q |> group_by([i], i.page) - |> select_merge([i], %{^key => selected_as(i.page, ^key), time_on_page: sum(i.time_on_page)}) + |> select_merge_as([i], %{key => i.page, time_on_page: sum(i.time_on_page)}) end defp group_imported_by(q, :country, key) do q |> group_by([i], i.country) |> where([i], i.country != "ZZ") - |> select_merge([i], %{^key => selected_as(i.country, ^key)}) + |> select_merge_as([i], %{key => i.country}) end defp group_imported_by(q, :region, key) do q |> group_by([i], i.region) |> where([i], i.region != "") - |> select_merge([i], %{^key => selected_as(i.region, ^key)}) + |> select_merge_as([i], %{key => i.region}) end defp group_imported_by(q, :city, key) do q |> group_by([i], i.city) |> where([i], i.city != 0 and not is_nil(i.city)) - |> select_merge([i], %{^key => selected_as(i.city, ^key)}) + |> select_merge_as([i], %{key => i.city}) end defp group_imported_by(q, dim, key) when dim in [:device, :browser] do q |> group_by([i], field(i, ^dim)) - |> select_merge([i], %{ - ^key => - selected_as( - fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim)), - ^key - ) + |> select_merge_as([i], %{ + key => fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim)) }) end defp group_imported_by(q, :browser_version, key) do q |> group_by([i], [i.browser_version]) - |> select_merge([i], %{ - ^key => - selected_as( - fragment( - "if(empty(?), ?, ?)", - i.browser_version, - @not_set, - i.browser_version - ), - ^key - ) + |> select_merge_as([i], %{ + key => fragment("if(empty(?), ?, ?)", i.browser_version, @not_set, i.browser_version) }) end defp group_imported_by(q, :os, key) do q |> group_by([i], i.operating_system) - |> select_merge([i], %{ - ^key => - selected_as( - fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system), - ^key - ) + |> select_merge_as([i], %{ + key => fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system) }) end defp group_imported_by(q, :os_version, key) do q |> group_by([i], [i.operating_system_version]) - |> select_merge([i], %{ - ^key => - selected_as( - fragment( - "if(empty(?), ?, ?)", - i.operating_system_version, - @not_set, - i.operating_system_version - ), - ^key + |> select_merge_as([i], %{ + key => + fragment( + "if(empty(?), ?, ?)", + i.operating_system_version, + @not_set, + i.operating_system_version ) }) end @@ -673,28 +640,28 @@ defmodule Plausible.Stats.Imported do defp group_imported_by(q, dim, key) when dim in [:entry_page, :exit_page] do q |> group_by([i], field(i, ^dim)) - |> select_merge([i], %{^key => selected_as(field(i, ^dim), ^key)}) + |> select_merge_as([i], %{key => field(i, ^dim)}) end defp group_imported_by(q, :name, key) do q |> group_by([i], i.name) - |> select_merge([i], %{^key => selected_as(i.name, ^key)}) + |> select_merge_as([i], %{key => i.name}) end defp group_imported_by(q, :url, key) do q |> group_by([i], i.link_url) - |> select_merge([i], %{ - ^key => selected_as(fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none), ^key) + |> select_merge_as([i], %{ + key => fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none) }) end defp group_imported_by(q, :path, key) do q |> group_by([i], i.path) - |> select_merge([i], %{ - ^key => selected_as(fragment("if(not empty(?), ?, ?)", i.path, i.path, @none), ^key) + |> select_merge_as([i], %{ + key => fragment("if(not empty(?), ?, ?)", i.path, i.path, @none) }) end @@ -705,23 +672,14 @@ defmodule Plausible.Stats.Imported do end defp select_joined_dimension(q, "visit:city", key) do - select_merge(q, [s, i], %{ - ^key => selected_as(fragment("greatest(?,?)", field(i, ^key), field(s, ^key)), ^key) + select_merge_as(q, [s, i], %{ + key => fragment("greatest(?,?)", field(i, ^key), field(s, ^key)) }) end defp select_joined_dimension(q, _dimension, key) do - select_merge(q, [s, i], %{ - ^key => - selected_as( - fragment( - "if(empty(?), ?, ?)", - field(s, ^key), - field(i, ^key), - field(s, ^key) - ), - ^key - ) + select_merge_as(q, [s, i], %{ + key => fragment("if(empty(?), ?, ?)", field(s, ^key), field(i, ^key), field(s, ^key)) }) end @@ -734,31 +692,31 @@ defmodule Plausible.Stats.Imported do defp select_joined_metrics(q, [:visits | rest]) do q - |> select_merge([s, i], %{visits: selected_as(s.visits + i.visits, :visits)}) + |> select_merge_as([s, i], %{visits: s.visits + i.visits}) |> select_joined_metrics(rest) end defp select_joined_metrics(q, [:visitors | rest]) do q - |> select_merge([s, i], %{visitors: selected_as(s.visitors + i.visitors, :visitors)}) + |> select_merge_as([s, i], %{visitors: s.visitors + i.visitors}) |> select_joined_metrics(rest) end defp select_joined_metrics(q, [:events | rest]) do q - |> select_merge([s, i], %{events: selected_as(s.events + i.events, :events)}) + |> select_merge_as([s, i], %{events: s.events + i.events}) |> select_joined_metrics(rest) end defp select_joined_metrics(q, [:pageviews | rest]) do q - |> select_merge([s, i], %{pageviews: selected_as(s.pageviews + i.pageviews, :pageviews)}) + |> select_merge_as([s, i], %{pageviews: s.pageviews + i.pageviews}) |> select_joined_metrics(rest) end defp select_joined_metrics(q, [:views_per_visit | rest]) do q - |> select_merge([s, i], %{ + |> select_merge_as([s, i], %{ views_per_visit: fragment( "if(? + ? > 0, round((? + ? * ?) / (? + ?), 2), 0)", @@ -776,7 +734,7 @@ defmodule Plausible.Stats.Imported do defp select_joined_metrics(q, [:bounce_rate | rest]) do q - |> select_merge([s, i], %{ + |> select_merge_as([s, i], %{ bounce_rate: fragment( "if(? + ? > 0, round(100 * (? + (? * ? / 100)) / (? + ?)), 0)", @@ -794,7 +752,7 @@ defmodule Plausible.Stats.Imported do defp select_joined_metrics(q, [:visit_duration | rest]) do q - |> select_merge([s, i], %{ + |> select_merge_as([s, i], %{ visit_duration: fragment( """ @@ -818,7 +776,7 @@ defmodule Plausible.Stats.Imported do defp select_joined_metrics(q, [:sample_percent | rest]) do q - |> select_merge([s, i], %{sample_percent: s.sample_percent}) + |> select_merge_as([s, i], %{sample_percent: s.sample_percent}) |> select_joined_metrics(rest) end @@ -831,10 +789,11 @@ defmodule Plausible.Stats.Imported do from(a in subquery(q1), full_join: b in subquery(q2), on: a.dim0 == b.dim0, - select: %{ - dim0: selected_as(fragment("if(? != 0, ?, ?)", a.dim0, a.dim0, b.dim0), :dim0) - } + select: %{} ) + |> select_merge_as([a, b], %{ + dim0: fragment("if(? != 0, ?, ?)", a.dim0, a.dim0, b.dim0) + }) |> select_joined_metrics(metrics) end end diff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.ex index d3f38a142..daea6791f 100644 --- a/lib/plausible/stats/sql/expression.ex +++ b/lib/plausible/stats/sql/expression.ex @@ -1,131 +1,253 @@ defmodule Plausible.Stats.SQL.Expression do @moduledoc """ This module is responsible for generating SQL/Ecto expressions - for dimensions used in query select, group_by and order_by. + for dimensions and metrics used in query SELECT statement. + + Each dimension and metric is tagged with with selected_as for easier + usage down the line. """ + use Plausible + use Plausible.Stats.SQL.Fragments + import Ecto.Query - use Plausible.Stats.Fragments + alias Plausible.Stats.{Query, SQL} @no_ref "Direct / None" @not_set "(not set)" - defmacrop field_or_blank_value(expr, empty_value, select_alias) do + defmacrop field_or_blank_value(key, expr, empty_value) do quote do - dynamic( - [t], - selected_as( - fragment("if(empty(?), ?, ?)", unquote(expr), unquote(empty_value), unquote(expr)), - ^unquote(select_alias) - ) - ) + wrap_alias([t], %{ + unquote(key) => + fragment("if(empty(?), ?, ?)", unquote(expr), unquote(empty_value), unquote(expr)) + }) end end - def dimension("time:hour", query, select_alias) do - dynamic( - [t], - selected_as( - fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone), - ^select_alias - ) - ) + def dimension(key, "time:hour", query) do + wrap_alias([t], %{ + key => fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone) + }) end - def dimension("time:day", query, select_alias) do - dynamic( - [t], - selected_as( - fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone), - ^select_alias - ) - ) + def dimension(key, "time:day", query) do + wrap_alias([t], %{ + key => fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone) + }) end - def dimension("time:month", query, select_alias) do - dynamic( - [t], - selected_as( - fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone), - ^select_alias - ) - ) + def dimension(key, "time:month", query) do + wrap_alias([t], %{ + key => fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone) + }) end - def dimension("event:name", _query, select_alias), - do: dynamic([t], selected_as(t.name, ^select_alias)) + def dimension(key, "event:name", _query), + do: wrap_alias([t], %{key => t.name}) - def dimension("event:page", _query, select_alias), - do: dynamic([t], selected_as(t.pathname, ^select_alias)) + def dimension(key, "event:page", _query), + do: wrap_alias([t], %{key => t.pathname}) - def dimension("event:hostname", _query, select_alias), - do: dynamic([t], selected_as(t.hostname, ^select_alias)) + def dimension(key, "event:hostname", _query), + do: wrap_alias([t], %{key => t.hostname}) - def dimension("event:props:" <> property_name, _query, select_alias) do - dynamic( - [t], - selected_as( + def dimension(key, "event:props:" <> property_name, _query) do + wrap_alias([t], %{ + key => fragment( "if(not empty(?), ?, '(none)')", get_by_key(t, :meta, ^property_name), get_by_key(t, :meta, ^property_name) - ), - ^select_alias - ) - ) + ) + }) end - def dimension("visit:entry_page", _query, select_alias), - do: dynamic([t], selected_as(t.entry_page, ^select_alias)) + def dimension(key, "visit:entry_page", _query), + do: wrap_alias([t], %{key => t.entry_page}) - def dimension("visit:exit_page", _query, select_alias), - do: dynamic([t], selected_as(t.exit_page, ^select_alias)) + def dimension(key, "visit:exit_page", _query), + do: wrap_alias([t], %{key => t.exit_page}) - def dimension("visit:utm_medium", _query, select_alias), - do: field_or_blank_value(t.utm_medium, @not_set, select_alias) + def dimension(key, "visit:utm_medium", _query), + do: field_or_blank_value(key, t.utm_medium, @not_set) - def dimension("visit:utm_source", _query, select_alias), - do: field_or_blank_value(t.utm_source, @not_set, select_alias) + def dimension(key, "visit:utm_source", _query), + do: field_or_blank_value(key, t.utm_source, @not_set) - def dimension("visit:utm_campaign", _query, select_alias), - do: field_or_blank_value(t.utm_campaign, @not_set, select_alias) + def dimension(key, "visit:utm_campaign", _query), + do: field_or_blank_value(key, t.utm_campaign, @not_set) - def dimension("visit:utm_content", _query, select_alias), - do: field_or_blank_value(t.utm_content, @not_set, select_alias) + def dimension(key, "visit:utm_content", _query), + do: field_or_blank_value(key, t.utm_content, @not_set) - def dimension("visit:utm_term", _query, select_alias), - do: field_or_blank_value(t.utm_term, @not_set, select_alias) + def dimension(key, "visit:utm_term", _query), + do: field_or_blank_value(key, t.utm_term, @not_set) - def dimension("visit:source", _query, select_alias), - do: field_or_blank_value(t.source, @no_ref, select_alias) + def dimension(key, "visit:source", _query), + do: field_or_blank_value(key, t.source, @no_ref) - def dimension("visit:referrer", _query, select_alias), - do: field_or_blank_value(t.referrer, @no_ref, select_alias) + def dimension(key, "visit:referrer", _query), + do: field_or_blank_value(key, t.referrer, @no_ref) - def dimension("visit:device", _query, select_alias), - do: field_or_blank_value(t.device, @not_set, select_alias) + def dimension(key, "visit:device", _query), + do: field_or_blank_value(key, t.device, @not_set) - def dimension("visit:os", _query, select_alias), - do: field_or_blank_value(t.os, @not_set, select_alias) + def dimension(key, "visit:os", _query), + do: field_or_blank_value(key, t.os, @not_set) - def dimension("visit:os_version", _query, select_alias), - do: field_or_blank_value(t.os_version, @not_set, select_alias) + def dimension(key, "visit:os_version", _query), + do: field_or_blank_value(key, t.os_version, @not_set) - def dimension("visit:browser", _query, select_alias), - do: field_or_blank_value(t.browser, @not_set, select_alias) + def dimension(key, "visit:browser", _query), + do: field_or_blank_value(key, t.browser, @not_set) - def dimension("visit:browser_version", _query, select_alias), - do: field_or_blank_value(t.browser_version, @not_set, select_alias) + def dimension(key, "visit:browser_version", _query), + do: field_or_blank_value(key, t.browser_version, @not_set) - def dimension("visit:country", _query, select_alias), - do: dynamic([t], selected_as(t.country, ^select_alias)) + def dimension(key, "visit:country", _query), + do: wrap_alias([t], %{key => t.country}) - def dimension("visit:region", _query, select_alias), - do: dynamic([t], selected_as(t.region, ^select_alias)) + def dimension(key, "visit:region", _query), + do: wrap_alias([t], %{key => t.region}) - def dimension("visit:city", _query, select_alias), - do: dynamic([t], selected_as(t.city, ^select_alias)) + def dimension(key, "visit:city", _query), + do: wrap_alias([t], %{key => t.city}) + + def event_metric(:pageviews) do + wrap_alias([e], %{ + pageviews: + fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name) + }) + end + + def event_metric(:events) do + wrap_alias([], %{ + events: fragment("toUInt64(round(count(*) * any(_sample_factor)))") + }) + end + + def event_metric(:visitors) do + wrap_alias([e], %{ + visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.user_id) + }) + end + + def event_metric(:visits) do + wrap_alias([e], %{ + visits: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.session_id) + }) + end + + on_ee do + def event_metric(:total_revenue) do + wrap_alias( + [e], + %{ + total_revenue: + fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount) + } + ) + end + + def event_metric(:average_revenue) do + wrap_alias( + [e], + %{ + average_revenue: + fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount) + } + ) + end + end + + def event_metric(:sample_percent) do + wrap_alias([], %{ + sample_percent: + fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)") + }) + end + + def event_metric(:percentage), do: %{} + def event_metric(:conversion_rate), do: %{} + def event_metric(:group_conversion_rate), do: %{} + def event_metric(:total_visitors), do: %{} + + def event_metric(unknown), do: raise("Unknown metric: #{unknown}") + + def session_metric(:bounce_rate, query) do + # :TRICKY: If page is passed to query, we only count bounce rate where users _entered_ at page. + event_page_filter = Query.get_filter(query, "event:page") + condition = SQL.WhereBuilder.build_condition(:entry_page, event_page_filter) + + wrap_alias([], %{ + bounce_rate: + fragment( + "toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))", + ^condition, + ^condition + ), + __internal_visits: fragment("toUInt32(sum(sign))") + }) + end + + def session_metric(:visits, _query) do + wrap_alias([s], %{ + visits: fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign) + }) + end + + def session_metric(:pageviews, _query) do + wrap_alias([s], %{ + pageviews: + fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews) + }) + end + + def session_metric(:events, _query) do + wrap_alias([s], %{ + events: fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events) + }) + end + + def session_metric(:visitors, _query) do + wrap_alias([s], %{ + visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", s.user_id) + }) + end + + def session_metric(:visit_duration, _query) do + wrap_alias([], %{ + visit_duration: + fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))"), + __internal_visits: fragment("toUInt32(sum(sign))") + }) + end + + def session_metric(:views_per_visit, _query) do + wrap_alias([s], %{ + views_per_visit: + fragment( + "ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)", + s.sign, + s.pageviews, + s.sign + ), + __internal_visits: fragment("toUInt32(sum(sign))") + }) + end + + def session_metric(:sample_percent, _query) do + wrap_alias([], %{ + sample_percent: + fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)") + }) + end + + def session_metric(:percentage, _query), do: %{} + def session_metric(:conversion_rate, _query), do: %{} + def session_metric(:group_conversion_rate, _query), do: %{} defmacro event_goal_join(events, page_regexes) do quote do diff --git a/lib/plausible/stats/fragments.ex b/lib/plausible/stats/sql/fragments.ex similarity index 65% rename from lib/plausible/stats/fragments.ex rename to lib/plausible/stats/sql/fragments.ex index 27f49839f..526682f9e 100644 --- a/lib/plausible/stats/fragments.ex +++ b/lib/plausible/stats/sql/fragments.ex @@ -1,4 +1,15 @@ -defmodule Plausible.Stats.Fragments do +defmodule Plausible.Stats.SQL.Fragments do + @moduledoc """ + Various macros and common SQL fragments used in Stats code. + """ + + defmacro __using__(_) do + quote do + import Plausible.Stats.SQL.Fragments + require Plausible.Stats.SQL.Fragments + end + end + defmacro uniq(user_id) do quote do fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", unquote(user_id)) @@ -56,21 +67,23 @@ defmodule Plausible.Stats.Fragments do `not_before` boundary is set to the past Saturday, which is before the weekstart, therefore the cap does not apply. - iex> this_wednesday = ~D[2022-11-09] - ...> past_saturday = ~D[2022-11-05] - ...> weekstart_not_before(this_wednesday, past_saturday) + ``` + > this_wednesday = ~D[2022-11-09] + > past_saturday = ~D[2022-11-05] + > weekstart_not_before(this_wednesday, past_saturday) ~D[2022-11-07] - + ``` In this other example, the fragment returns Tuesday and not the weekstart. The `not_before` boundary is set to Tuesday, which is past the weekstart, therefore the cap applies. - iex> this_wednesday = ~D[2022-11-09] - ...> this_tuesday = ~D[2022-11-08] - ...> weekstart_not_before(this_wednesday, this_tuesday) + ``` + > this_wednesday = ~D[2022-11-09] + > this_tuesday = ~D[2022-11-08] + > weekstart_not_before(this_wednesday, this_tuesday) ~D[2022-11-08] - + ``` """ defmacro weekstart_not_before(date, not_before) do quote do @@ -85,7 +98,7 @@ defmodule Plausible.Stats.Fragments do end @doc """ - Same as Plausible.Stats.Fragments.weekstart_not_before/2 but converts dates to + Same as Plausible.Stats.SQL.Fragments.weekstart_not_before/2 but converts dates to the specified timezone. """ defmacro weekstart_not_before(date, not_before, timezone) do @@ -143,9 +156,51 @@ defmodule Plausible.Stats.Fragments do def meta_value_column(:meta), do: :"meta.value" def meta_value_column(:entry_meta), do: :"entry_meta.value" - defmacro __using__(_) do + @doc """ + Convenience Ecto macro for wrapping a map passed to select_merge_as such that each + expression gets wrapped in dynamic and set as selected_as. + + ### Examples + + iex> wrap_alias([t], %{ foo: t.column }) |> expand_macro_once + "%{foo: dynamic([t], selected_as(t.column, :foo))}" + """ + defmacro wrap_alias(binding, map_literal) do + update_literal_map_values(map_literal, fn {key, expr} -> + key_expr = + if Macro.quoted_literal?(key) do + key + else + quote(do: ^unquote(key)) + end + + quote(do: dynamic(unquote(binding), selected_as(unquote(expr), unquote(key_expr)))) + end) + end + + @doc """ + Convenience Ecto macro for wrapping select_merge where each value gets in turn passed to selected_as. + + ### Examples + + iex> select_merge_as(q, [t], %{ foo: t.column }) |> expand_macro_once + "select_merge(q, [], ^wrap_alias([t], %{foo: t.column}))" + """ + defmacro select_merge_as(q, binding, map_literal) do quote do - import Plausible.Stats.Fragments + select_merge(unquote(q), [], ^wrap_alias(unquote(binding), unquote(map_literal))) end end + + defp update_literal_map_values({:%{}, ctx, keyword_list}, mapper_fn) do + { + :%{}, + ctx, + Enum.map(keyword_list, fn {key, expr} -> + {key, mapper_fn.({key, expr})} + end) + } + end + + defp update_literal_map_values(ast, _), do: ast end diff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.ex index 7e5e49a88..8cf92725b 100644 --- a/lib/plausible/stats/sql/query_builder.ex +++ b/lib/plausible/stats/sql/query_builder.ex @@ -2,12 +2,13 @@ defmodule Plausible.Stats.SQL.QueryBuilder do @moduledoc false use Plausible + use Plausible.Stats.SQL.Fragments import Ecto.Query import Plausible.Stats.Imported import Plausible.Stats.Util - alias Plausible.Stats.{Base, Query, QueryOptimizer, TableDecider, Filters} + alias Plausible.Stats.{Base, Filters, Query, QueryOptimizer, TableDecider, SQL} alias Plausible.Stats.SQL.Expression require Plausible.Stats.SQL.Expression @@ -30,7 +31,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do q = from( e in "events_v2", - where: ^Filters.WhereBuilder.build(:events, site, events_query), + where: ^SQL.WhereBuilder.build(:events, site, events_query), select: ^Base.select_event_metrics(events_query.metrics) ) @@ -73,7 +74,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do q = from( e in "sessions_v2", - where: ^Filters.WhereBuilder.build(:sessions, site, sessions_query), + where: ^SQL.WhereBuilder.build(:sessions, site, sessions_query), select: ^Base.select_session_metrics(sessions_query.metrics, sessions_query) ) @@ -94,7 +95,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do if Query.has_event_filters?(query) do events_q = from(e in "events_v2", - where: ^Filters.WhereBuilder.build(:events, site, query), + where: ^SQL.WhereBuilder.build(:events, site, query), select: %{ session_id: fragment("DISTINCT ?", e.session_id), _sample_factor: fragment("_sample_factor") @@ -135,7 +136,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do key = shortname(query, dimension) q - |> select_merge(^%{key => Expression.dimension(dimension, query, key)}) + |> select_merge_as([], Expression.dimension(key, dimension, query)) |> group_by([], selected_as(^key)) end @@ -147,9 +148,9 @@ defmodule Plausible.Stats.SQL.QueryBuilder do order_by( q, [t], - ^{ - order_direction, - dynamic([], selected_as(^shortname(query, metric_or_dimension))) + { + ^order_direction, + selected_as(^shortname(query, metric_or_dimension)) } ) end @@ -157,19 +158,11 @@ defmodule Plausible.Stats.SQL.QueryBuilder do defmacrop select_join_fields(q, query, list, table_name) do quote do Enum.reduce(unquote(list), unquote(q), fn metric_or_dimension, q -> - select_merge( - q, - ^%{ - shortname(unquote(query), metric_or_dimension) => - dynamic( - [e, s], - selected_as( - field(unquote(table_name), ^shortname(unquote(query), metric_or_dimension)), - ^shortname(unquote(query), metric_or_dimension) - ) - ) - } - ) + key = shortname(unquote(query), metric_or_dimension) + + select_merge_as(q, [e, s], %{ + key => field(unquote(table_name), ^key) + }) end) end end @@ -185,21 +178,17 @@ defmodule Plausible.Stats.SQL.QueryBuilder do |> Query.set_dimensions([]) q - |> select_merge( - ^%{ - total_visitors: Base.total_visitors_subquery(site, total_query, query.include_imported) - } + |> select_merge_as( + [], + Base.total_visitors_subquery(site, total_query, query.include_imported) ) - |> select_merge([e], %{ + |> select_merge_as([e], %{ conversion_rate: - selected_as( - fragment( - "if(? > 0, round(? / ? * 100, 1), 0)", - selected_as(:__total_visitors), - selected_as(:visitors), - selected_as(:__total_visitors) - ), - :conversion_rate + fragment( + "if(? > 0, round(? / ? * 100, 1), 0)", + selected_as(:total_visitors), + selected_as(:visitors), + selected_as(:total_visitors) ) }) else @@ -228,21 +217,18 @@ defmodule Plausible.Stats.SQL.QueryBuilder do from(e in subquery(q), left_join: c in subquery(build(group_totals_query, site)), - on: ^build_group_by_join(query), - select_merge: %{ - total_visitors: c.visitors, - group_conversion_rate: - selected_as( - fragment( - "if(? > 0, round(? / ? * 100, 1), 0)", - c.visitors, - e.visitors, - c.visitors - ), - :group_conversion_rate - ) - } + on: ^build_group_by_join(query) ) + |> select_merge_as([e, c], %{ + total_visitors: c.visitors, + group_conversion_rate: + fragment( + "if(? > 0, round(? / ? * 100, 1), 0)", + c.visitors, + e.visitors, + c.visitors + ) + }) |> select_join_fields(query, query.dimensions, e) |> select_join_fields(query, List.delete(query.metrics, :group_conversion_rate), e) else diff --git a/lib/plausible/stats/filters/where_builder.ex b/lib/plausible/stats/sql/where_builder.ex similarity index 99% rename from lib/plausible/stats/filters/where_builder.ex rename to lib/plausible/stats/sql/where_builder.ex index 257d4257a..71bcdad49 100644 --- a/lib/plausible/stats/filters/where_builder.ex +++ b/lib/plausible/stats/sql/where_builder.ex @@ -1,4 +1,4 @@ -defmodule Plausible.Stats.Filters.WhereBuilder do +defmodule Plausible.Stats.SQL.WhereBuilder do @moduledoc """ A module for building am ecto where clause of a query out of a query. """ @@ -8,7 +8,7 @@ defmodule Plausible.Stats.Filters.WhereBuilder do alias Plausible.Stats.Query - use Plausible.Stats.Fragments + use Plausible.Stats.SQL.Fragments require Logger diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex index 69e3dc528..4914023bf 100644 --- a/lib/plausible/stats/timeseries.ex +++ b/lib/plausible/stats/timeseries.ex @@ -4,7 +4,7 @@ defmodule Plausible.Stats.Timeseries do alias Plausible.Stats.{Query, Util, Imported} import Plausible.Stats.{Base} import Ecto.Query - use Plausible.Stats.Fragments + use Plausible.Stats.SQL.Fragments @typep metric :: :pageviews diff --git a/lib/plausible/stats/util.ex b/lib/plausible/stats/util.ex index 60bdd7a93..de1411f63 100644 --- a/lib/plausible/stats/util.ex +++ b/lib/plausible/stats/util.ex @@ -6,7 +6,6 @@ defmodule Plausible.Stats.Util do @manually_removable_metrics [ :__internal_visits, :visitors, - :__total_visitors, :__breakdown_value, :total_visitors ] diff --git a/test/plausible/stats/sql/fragments_test.exs b/test/plausible/stats/sql/fragments_test.exs new file mode 100644 index 000000000..0932d7e59 --- /dev/null +++ b/test/plausible/stats/sql/fragments_test.exs @@ -0,0 +1,10 @@ +defmodule Plausible.Stats.SQL.FragmentsTest do + use ExUnit.Case, async: true + use Plausible.Stats.SQL.Fragments + + defmacro expand_macro_once(ast) do + ast |> Macro.expand_once(__ENV__) |> Macro.to_string() + end + + doctest Plausible.Stats.SQL.Fragments +end