diff --git a/CHANGELOG.md b/CHANGELOG.md index 065196ce6..75f271eeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ All notable changes to this project will be documented in this file. - Increase hourly request limit for API keys in CE from 600 to 1000000 (practically removing the limit) plausible/analytics#4200 - Make TCP connections try IPv6 first with IPv4 fallback in CE plausible/analytics#4245 - `is` and `is not` filters in dashboard no longer support wildcards. Use contains/does not contain filter instead. +- `bounce_rate` metric now returns 0 instead of null for event:page breakdown when page has never been entry page. ### Fixed diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 3b3dfd2e3..5354833a1 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -250,6 +250,8 @@ defmodule Plausible.Stats.Base do end defp select_session_metric(:percentage, _query), do: %{} + defp select_session_metric(:conversion_rate, _query), do: %{} + defp select_session_metric(:group_conversion_rate, _query), do: %{} def filter_converted_sessions(db_query, site, query) do if Query.has_event_filters?(query) do diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index d68e9fcad..a0bdaa150 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -3,344 +3,63 @@ defmodule Plausible.Stats.Breakdown do use Plausible use Plausible.Stats.Fragments - import Plausible.Stats.{Base, Imported} + import Plausible.Stats.Base import Ecto.Query - alias Plausible.Stats.{Query, Util, TableDecider} + alias Plausible.Stats.{Query, QueryOptimizer, QueryResult, SQL} + alias Plausible.Stats.Filters.QueryParser - @no_ref "Direct / None" - @not_set "(not set)" + def breakdown(site, %Query{dimensions: [dimension]} = query, metrics, pagination, _opts \\ []) do + transformed_metrics = transform_metrics(metrics, dimension) - @session_metrics [:bounce_rate, :visit_duration] - - @revenue_metrics on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: []) - - @event_metrics [:visits, :visitors, :pageviews, :events, :percentage] ++ @revenue_metrics - - # These metrics can be asked from the `breakdown/5` function, - # but they are different from regular metrics such as `visitors`, - # or `bounce_rate` - we cannot currently "select them" directly in - # the db queries. Instead, we need to artificially append them to - # the breakdown results later on. - @computed_metrics [:conversion_rate, :total_visitors] - - def breakdown(site, query, metrics, pagination, opts \\ []) - - def breakdown( - site, - %Query{dimensions: ["event:goal"]} = query, - metrics, - pagination, - opts - ) do - site = Plausible.Repo.preload(site, :goals) - - {event_goals, pageview_goals} = Enum.split_with(site.goals, & &1.event_name) - events = Enum.map(event_goals, & &1.event_name) - - event_query = - query - |> Query.put_filter([:is, "event:name", events]) - |> Query.set_dimensions(["event:name"]) - - if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics) - - no_revenue = {nil, metrics -- @revenue_metrics} - - {revenue_goals, metrics} = - on_ee do - if Plausible.Billing.Feature.RevenueGoals.enabled?(site) do - revenue_goals = Enum.filter(event_goals, &Plausible.Goal.Revenue.revenue?/1) - metrics = if Enum.empty?(revenue_goals), do: metrics -- @revenue_metrics, else: metrics - - {revenue_goals, metrics} - else - no_revenue - end - else - no_revenue - end - - metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics - - event_q = - if Enum.any?(event_goals) do - site - |> breakdown_events(event_query, metrics_to_select) - |> apply_pagination(pagination) - else - nil - end - - page_q = - if Enum.any?(pageview_goals) do - page_query = Query.set_dimensions(query, ["event:page"]) - - page_exprs = Enum.map(pageview_goals, & &1.page_path) - page_regexes = Enum.map(page_exprs, &page_regex/1) - - select_columns = metrics_to_select |> select_event_metrics |> mark_revenue_as_nil - - from(e in base_event_query(site, page_query), - order_by: [desc: fragment("uniq(?)", e.user_id)], - where: - fragment( - "notEmpty(multiMatchAllIndices(?, ?) as indices)", - e.pathname, - ^page_regexes - ) and e.name == "pageview", - array_join: index in fragment("indices"), - group_by: index, - select: %{ - name: fragment("concat('Visit ', ?[?])", ^page_exprs, index) - } - ) - |> select_merge(^select_columns) - |> merge_imported_pageview_goals(site, page_query, page_exprs, metrics_to_select) - |> apply_pagination(pagination) - else - nil - end - - full_q = - case {event_q, page_q} do - {nil, nil} -> - nil - - {event_q, nil} -> - event_q - - {nil, page_q} -> - page_q - - {event_q, page_q} -> - from( - e in subquery(union_all(event_q, ^page_q)), - order_by: [desc: e.visitors] - ) - |> apply_pagination(pagination) - end - - if full_q do - full_q - |> maybe_add_conversion_rate(site, query, metrics) - |> ClickhouseRepo.all() - |> transform_keys(%{name: :goal}) - |> cast_revenue_metrics_to_money(revenue_goals) - |> Util.keep_requested_metrics(metrics) - else - [] - end - end - - def breakdown( - site, - %Query{dimensions: ["event:props:" <> custom_prop]} = query, - metrics, - pagination, - opts - ) do - {currency, metrics} = - on_ee do - Plausible.Stats.Goal.Revenue.get_revenue_tracking_currency(site, query, metrics) - else - {nil, metrics} - end - - metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics - - if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics) - - breakdown_events(site, query, metrics_to_select) - |> maybe_add_conversion_rate(site, query, metrics) - |> paginate_and_execute(metrics, pagination) - |> transform_keys(%{breakdown_prop_value: custom_prop}) - |> Enum.map(&cast_revenue_metrics_to_money(&1, currency)) - end - - def breakdown(site, %Query{dimensions: ["event:page"]} = query, metrics, pagination, opts) do - event_metrics = - metrics - |> Util.maybe_add_visitors_metric() - |> Enum.filter(&(&1 in @event_metrics)) - - if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics) - - event_result = - site - |> breakdown_events(query, event_metrics) - |> maybe_add_group_conversion_rate(&breakdown_events/3, site, query, metrics) - |> paginate_and_execute(metrics, pagination) - |> maybe_add_time_on_page(site, query, metrics) - - session_metrics = Enum.filter(metrics, &(&1 in @session_metrics)) - - entry_page_query = - case event_result do - [] -> - query - - pages -> - query - |> Query.remove_filters(["event:page"]) - |> Query.put_filter([:is, "visit:entry_page", Enum.map(pages, & &1[:page])]) - |> Query.set_dimensions(["visit:entry_page"]) - end - - if Enum.any?(event_metrics) && Enum.empty?(event_result) do - [] - else - {limit, _page} = pagination - - session_result = - breakdown_sessions(site, entry_page_query, session_metrics) - |> paginate_and_execute(session_metrics, {limit, 1}) - |> transform_keys(%{entry_page: :page}) - - metrics = metrics ++ [:page] - - zip_results( - event_result, - session_result, - :page, - metrics + query_with_metrics = + Query.set( + query, + metrics: transformed_metrics, + order_by: infer_order_by(transformed_metrics, dimension), + dimensions: transform_dimensions(dimension), + filters: query.filters ++ dimension_filters(dimension), + preloaded_goals: QueryParser.preload_goals_if_needed(site, query.filters, [dimension]), + v2: true, + # Allow pageview and event metrics to be queried off of sessions table + legacy_breakdown: true ) - |> Enum.map(&Map.take(&1, metrics)) - end - end + |> QueryOptimizer.optimize() - def breakdown(site, %Query{dimensions: ["event:name"]} = query, metrics, pagination, opts) do - if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics) + q = SQL.QueryBuilder.build(query_with_metrics, site) - breakdown_events(site, query, metrics) - |> paginate_and_execute(metrics, pagination) - end - - def breakdown(site, query, metrics, pagination, opts) do - query = maybe_update_breakdown_filters(query) - if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics) - - metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics - - case breakdown_table(query, metrics) do - :session -> - breakdown_sessions(site, query, metrics_to_select) - |> maybe_add_group_conversion_rate(&breakdown_sessions/3, site, query, metrics) - |> paginate_and_execute(metrics, pagination) - - :event -> - breakdown_events(site, query, metrics_to_select) - |> maybe_add_group_conversion_rate(&breakdown_events/3, site, query, metrics) - |> paginate_and_execute(metrics, pagination) - end - end - - defp maybe_update_breakdown_filters(%Query{dimensions: [visit_entry_prop]} = query) - when visit_entry_prop in [ - "visit:source", - "visit:entry_page", - "visit:utm_medium", - "visit:utm_source", - "visit:utm_campaign", - "visit:utm_content", - "visit:utm_term", - "visit:entry_page", - "visit:referrer" - ] do - update_hostname_filter_prop(query, "visit:entry_page_hostname") - end - - defp maybe_update_breakdown_filters(%Query{dimensions: ["visit:exit_page"]} = query) do - update_hostname_filter_prop(query, "visit:exit_page_hostname") - end - - defp maybe_update_breakdown_filters(query) do - query - end - - defp update_hostname_filter_prop(query, visit_prop) do - case Query.get_filter(query, "event:hostname") do - nil -> - query - - [op, "event:hostname", value] -> - query - |> Query.put_filter([op, visit_prop, value]) - end - end - - # Backwards compatibility - defp breakdown_table(%Query{experimental_reduced_joins?: false}, _), do: :session - - defp breakdown_table(%Query{dimensions: ["visit:entry_page"]}, _metrics), do: :session - defp breakdown_table(%Query{dimensions: ["visit:entry_page_hostname"]}, _metrics), do: :session - defp breakdown_table(%Query{dimensions: ["visit:exit_page"]}, _metrics), do: :session - defp breakdown_table(%Query{dimensions: ["visit:exit_page_hostname"]}, _metrics), do: :session - - defp breakdown_table(%Query{dimensions: [_dimension]} = query, metrics) do - {_, session_metrics, _} = TableDecider.partition_metrics(metrics, query) - - if not Enum.empty?(session_metrics) do - :session - else - :event - end - end - - defp zip_results(event_result, session_result, property, metrics) do - null_row = Enum.map(metrics, fn metric -> {metric, nil} end) |> Enum.into(%{}) - - prop_values = - Enum.map(event_result ++ session_result, fn row -> row[property] end) - |> Enum.uniq() - - Enum.map(prop_values, fn value -> - event_row = Enum.find(event_result, fn row -> row[property] == value end) || %{} - session_row = Enum.find(session_result, fn row -> row[property] == value end) || %{} - - null_row - |> Map.merge(event_row) - |> Map.merge(session_row) - end) - |> sort_results(metrics) - end - - defp breakdown_sessions(site, %Query{dimensions: [dimension]} = query, metrics) do - from(s in query_sessions(site, query), - order_by: [desc: fragment("uniq(?)", s.user_id)], - select: ^select_session_metrics(metrics, query) - ) - |> filter_converted_sessions(site, query) - |> do_group_by(dimension) - |> merge_imported(site, query, metrics) - |> add_percentage_metric(site, query, metrics) - end - - defp breakdown_events(site, %Query{dimensions: [dimension]} = query, metrics) do - from(e in base_event_query(site, query), - order_by: [desc: fragment("uniq(?)", e.user_id)], - select: %{} - ) - |> do_group_by(dimension) - |> select_merge(^select_event_metrics(metrics)) - |> merge_imported(site, query, metrics) - |> add_percentage_metric(site, query, metrics) - end - - defp paginate_and_execute(_, [], _), do: [] - - defp paginate_and_execute(q, metrics, pagination) do q |> apply_pagination(pagination) |> ClickhouseRepo.all() - |> Util.keep_requested_metrics(metrics) + |> QueryResult.from(query_with_metrics) + |> build_breakdown_result(query_with_metrics, metrics) + |> maybe_add_time_on_page(site, query_with_metrics, metrics) + |> update_currency_metrics(site, query_with_metrics) end + defp build_breakdown_result(query_result, query, metrics) do + query_result.results + |> Enum.map(fn %{dimensions: dimensions, metrics: entry_metrics} -> + dimension_map = + query.dimensions |> Enum.map(&result_key/1) |> Enum.zip(dimensions) |> Enum.into(%{}) + + metrics_map = Enum.zip(metrics, entry_metrics) |> Enum.into(%{}) + + Map.merge(dimension_map, metrics_map) + end) + end + + defp result_key("event:props:" <> custom_property), do: custom_property + defp result_key("event:" <> key), do: key |> String.to_existing_atom() + defp result_key("visit:" <> key), do: key |> String.to_existing_atom() + defp result_key(dimension), do: dimension + defp maybe_add_time_on_page(event_results, site, query, metrics) do - if :time_on_page in metrics do + if query.dimensions == ["event:page"] and :time_on_page in metrics do pages = Enum.map(event_results, & &1[:page]) time_on_page_result = breakdown_time_on_page(site, query, pages) - Enum.map(event_results, fn row -> + event_results + |> Enum.map(fn row -> Map.put(row, :time_on_page, time_on_page_result[row[:page]]) end) else @@ -431,346 +150,60 @@ defmodule Plausible.Stats.Breakdown do |> Map.new() end - defp do_group_by( - %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q, - "event:props:" <> prop - ) do - from( - e in q, - select_merge: %{ - breakdown_prop_value: - selected_as( - fragment( - "if(not empty(?), ?, '(none)')", - get_by_key(e, :meta, ^prop), - get_by_key(e, :meta, ^prop) - ), - :breakdown_prop_value - ) - }, - group_by: selected_as(:breakdown_prop_value), - order_by: {:asc, selected_as(:breakdown_prop_value)} - ) - end + defp transform_metrics(metrics, dimension) do + metrics = + if is_nil(metric_to_order_by(metrics)) do + metrics ++ [:visitors] + else + metrics + end - defp do_group_by( - %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q, - "event:name" - ) do - from( - e in q, - group_by: e.name, - select_merge: %{name: e.name}, - order_by: {:asc, e.name} - ) - end - - defp do_group_by( - %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q, - "event:page" - ) do - from( - e in q, - group_by: e.pathname, - select_merge: %{page: e.pathname}, - order_by: {:asc, e.pathname} - ) - end - - defp do_group_by(q, "visit:source") do - from( - s in q, - group_by: s.source, - select_merge: %{ - source: fragment("if(empty(?), ?, ?)", s.source, @no_ref, s.source) - }, - order_by: {:asc, s.source} - ) - end - - defp do_group_by(q, "visit:country") do - from( - s in q, - where: s.country != "\0\0" and s.country != "ZZ", - group_by: s.country, - select_merge: %{country: s.country}, - order_by: {:asc, s.country} - ) - end - - defp do_group_by(q, "visit:region") do - from( - s in q, - where: s.region != "", - group_by: s.region, - select_merge: %{region: s.region}, - order_by: {:asc, s.region} - ) - end - - defp do_group_by(q, "visit:city") do - from( - s in q, - where: s.city != 0, - group_by: s.city, - select_merge: %{city: s.city}, - order_by: {:asc, s.city} - ) - end - - defp do_group_by(q, "visit:entry_page") do - from( - s in q, - # Sessions without pageviews don't get entry_page assigned, hence they should get ignored - where: s.entry_page != "", - group_by: s.entry_page, - select_merge: %{entry_page: s.entry_page}, - order_by: {:asc, s.entry_page} - ) - end - - defp do_group_by(q, "visit:exit_page") do - from( - s in q, - # Sessions without pageviews don't get entry_page assigned, hence they should get ignored - where: s.entry_page != "", - group_by: s.exit_page, - select_merge: %{exit_page: s.exit_page}, - order_by: {:asc, s.exit_page} - ) - end - - defp do_group_by(q, "visit:referrer") do - from( - s in q, - group_by: s.referrer, - select_merge: %{ - referrer: fragment("if(empty(?), ?, ?)", s.referrer, @no_ref, s.referrer) - }, - order_by: {:asc, s.referrer} - ) - end - - defp do_group_by(q, "visit:utm_medium") do - from( - s in q, - where: fragment("not empty(?)", s.utm_medium), - group_by: s.utm_medium, - select_merge: %{ - utm_medium: s.utm_medium - } - ) - end - - defp do_group_by(q, "visit:utm_source") do - from( - s in q, - where: fragment("not empty(?)", s.utm_source), - group_by: s.utm_source, - select_merge: %{ - utm_source: s.utm_source - } - ) - end - - defp do_group_by(q, "visit:utm_campaign") do - from( - s in q, - where: fragment("not empty(?)", s.utm_campaign), - group_by: s.utm_campaign, - select_merge: %{ - utm_campaign: s.utm_campaign - } - ) - end - - defp do_group_by(q, "visit:utm_content") do - from( - s in q, - where: fragment("not empty(?)", s.utm_content), - group_by: s.utm_content, - select_merge: %{ - utm_content: s.utm_content - } - ) - end - - defp do_group_by(q, "visit:utm_term") do - from( - s in q, - where: fragment("not empty(?)", s.utm_term), - group_by: s.utm_term, - select_merge: %{ - utm_term: s.utm_term - } - ) - end - - defp do_group_by(q, "visit:device") do - from( - s in q, - group_by: s.device, - select_merge: %{ - device: fragment("if(empty(?), ?, ?)", s.device, @not_set, s.device) - }, - order_by: {:asc, s.device} - ) - end - - defp do_group_by(q, "visit:os") do - from( - s in q, - group_by: s.os, - select_merge: %{ - os: fragment("if(empty(?), ?, ?)", s.os, @not_set, s.os) - }, - order_by: {:asc, s.os} - ) - end - - defp do_group_by(q, "visit:os_version") do - from( - s in q, - group_by: [s.os, s.os_version], - select_merge: %{ - os: fragment("if(empty(?), ?, ?)", s.os, @not_set, s.os), - os_version: - fragment( - "if(empty(?), ?, ?)", - s.os_version, - @not_set, - s.os_version - ) - }, - order_by: {:asc, s.os_version} - ) - end - - defp do_group_by(q, "visit:browser") do - from( - s in q, - group_by: s.browser, - select_merge: %{ - browser: fragment("if(empty(?), ?, ?)", s.browser, @not_set, s.browser) - }, - order_by: {:asc, s.browser} - ) - end - - defp do_group_by(q, "visit:browser_version") do - from( - s in q, - group_by: [s.browser, s.browser_version], - select_merge: %{ - browser: fragment("if(empty(?), ?, ?)", s.browser, @not_set, s.browser), - browser_version: - fragment("if(empty(?), ?, ?)", s.browser_version, @not_set, s.browser_version) - }, - order_by: {:asc, s.browser_version} - ) - end - - defp group_by_field_names("event:props:" <> _prop), do: [:name] - defp group_by_field_names("visit:os_version"), do: [:os, :os_version] - defp group_by_field_names("visit:browser_version"), do: [:browser, :browser_version] - - defp group_by_field_names(property), do: [Plausible.Stats.Filters.without_prefix(property)] - - defp on_matches_group_by(fields) do - Enum.reduce(fields, nil, &fields_equal/2) - end - - defp outer_order_by(fields) do - Enum.map(fields, fn field_name -> {:asc, dynamic([q], field(q, ^field_name))} end) - end - - defp fields_equal(field_name, nil), - do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name)) - - defp fields_equal(field_name, condition), - do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name) and ^condition) - - defp sort_results(results, metrics) do - Enum.sort_by( - results, - fn entry -> - case entry[sorting_key(metrics)] do - nil -> 0 - n -> n - end - end, - :desc - ) - end - - # This function injects a conversion_rate metric into - # a breakdown query. It is calculated as X / Y, where: - # - # * X is the number of conversions for a breakdown - # result (conversion = number of visitors who - # completed the filtered goal with the filtered - # custom properties). - # - # * Y is the number of all visitors for this breakdown - # result without the `event:goal` and `event:props:*` - # filters. - defp maybe_add_group_conversion_rate( - q, - breakdown_fn, - site, - %Query{dimensions: [dimension]} = query, - metrics - ) do - if :conversion_rate in metrics do - breakdown_total_visitors_query = - query |> Query.remove_filters(["event:goal", "event:props"]) - - breakdown_total_visitors_q = - breakdown_fn.(site, breakdown_total_visitors_query, [:visitors]) - - from(e in subquery(q), - left_join: c in subquery(breakdown_total_visitors_q), - on: ^on_matches_group_by(group_by_field_names(dimension)), - select_merge: %{ - total_visitors: c.visitors, - conversion_rate: - fragment( - "if(? > 0, round(? / ? * 100, 1), 0)", - c.visitors, - e.visitors, - c.visitors - ) - }, - order_by: [desc: e.visitors], - order_by: ^outer_order_by(group_by_field_names(dimension)) - ) - else - q - end - end - - # When querying custom event goals and pageviewgoals together, UNION ALL is used - # so the same fields must be present on both sides of the union. This change to the - # query will ensure that we don't unnecessarily read revenue column for pageview goals - defp mark_revenue_as_nil(select_columns) do - select_columns - |> Map.replace(:total_revenue, nil) - |> Map.replace(:average_revenue, nil) - end - - defp sorting_key(metrics) do - if Enum.member?(metrics, :visitors), do: :visitors, else: List.first(metrics) - end - - defp transform_keys(results, keys_to_replace) do - Enum.map(results, fn map -> - Enum.map(map, fn {key, val} -> - {Map.get(keys_to_replace, key, key), val} - end) - |> Enum.into(%{}) + Enum.map(metrics, fn metric -> + case {metric, dimension} do + {:conversion_rate, "event:props:" <> _} -> :conversion_rate + {:conversion_rate, "event:goal"} -> :conversion_rate + {:conversion_rate, _} -> :group_conversion_rate + _ -> metric + end end) end + defp infer_order_by(metrics, "event:goal"), do: [{metric_to_order_by(metrics), :desc}] + + defp infer_order_by(metrics, dimension), + do: [{metric_to_order_by(metrics), :desc}, {dimension, :asc}] + + defp metric_to_order_by(metrics) do + Enum.find(metrics, &(&1 != :time_on_page)) + end + + def transform_dimensions("visit:browser_version"), + do: ["visit:browser", "visit:browser_version"] + + def transform_dimensions("visit:os_version"), do: ["visit:os", "visit:os_version"] + def transform_dimensions(dimension), do: [dimension] + + @filter_dimensions_not %{ + "visit:city" => [0], + "visit:country" => ["\0\0", "ZZ"], + "visit:region" => [""], + "visit:utm_medium" => [""], + "visit:utm_source" => [""], + "visit:utm_campaign" => [""], + "visit:utm_content" => [""], + "visit:utm_term" => [""], + "visit:entry_page" => [""], + "visit:exit_page" => [""] + } + + @extra_filter_dimensions Map.keys(@filter_dimensions_not) + + defp dimension_filters(dimension) when dimension in @extra_filter_dimensions do + [[:is_not, dimension, Map.get(@filter_dimensions_not, dimension)]] + end + + defp dimension_filters(_), do: [] + defp apply_pagination(q, {limit, page}) do offset = (page - 1) * limit @@ -780,10 +213,39 @@ defmodule Plausible.Stats.Breakdown do end on_ee do - defp cast_revenue_metrics_to_money(results, revenue_goals) do - Plausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(results, revenue_goals) + defp update_currency_metrics(results, site, %Query{dimensions: ["event:goal"]}) do + site = Plausible.Repo.preload(site, :goals) + + {event_goals, _pageview_goals} = Enum.split_with(site.goals, & &1.event_name) + revenue_goals = Enum.filter(event_goals, &Plausible.Goal.Revenue.revenue?/1) + + if length(revenue_goals) > 0 and Plausible.Billing.Feature.RevenueGoals.enabled?(site) do + Plausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(results, revenue_goals) + else + remove_revenue_metrics(results) + end + end + + defp update_currency_metrics(results, site, query) do + {currency, _metrics} = + Plausible.Stats.Goal.Revenue.get_revenue_tracking_currency(site, query, query.metrics) + + if currency do + results + |> Enum.map(&Plausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(&1, currency)) + else + remove_revenue_metrics(results) + end end else - defp cast_revenue_metrics_to_money(results, _revenue_goals), do: results + defp update_currency_metrics(results, _site, _query), do: remove_revenue_metrics(results) + end + + defp remove_revenue_metrics(results) do + Enum.map(results, fn map -> + map + |> Map.delete(:total_revenue) + |> Map.delete(:average_revenue) + end) end end diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.ex index 018447890..3f802dcb6 100644 --- a/lib/plausible/stats/filters/query_parser.ex +++ b/lib/plausible/stats/filters/query_parser.ex @@ -4,6 +4,7 @@ defmodule Plausible.Stats.Filters.QueryParser do alias Plausible.Stats.TableDecider alias Plausible.Stats.Filters alias Plausible.Stats.Query + alias Plausible.Stats.Metrics def parse(site, params, now \\ nil) when is_map(params) do with {:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])), @@ -44,17 +45,12 @@ defmodule Plausible.Stats.Filters.QueryParser do defp parse_metrics(_invalid_metrics), do: {:error, "Invalid metrics passed"} - defp parse_metric("time_on_page"), do: {:ok, :time_on_page} - defp parse_metric("conversion_rate"), do: {:ok, :conversion_rate} - defp parse_metric("group_conversion_rate"), do: {:ok, :group_conversion_rate} - defp parse_metric("visitors"), do: {:ok, :visitors} - defp parse_metric("pageviews"), do: {:ok, :pageviews} - defp parse_metric("events"), do: {:ok, :events} - defp parse_metric("visits"), do: {:ok, :visits} - defp parse_metric("bounce_rate"), do: {:ok, :bounce_rate} - defp parse_metric("visit_duration"), do: {:ok, :visit_duration} - defp parse_metric("views_per_visit"), do: {:ok, :views_per_visit} - defp parse_metric(unknown_metric), do: {:error, "Unknown metric '#{inspect(unknown_metric)}'"} + defp parse_metric(metric_str) do + case Metrics.from_string(metric_str) do + {:ok, metric} -> {:ok, metric} + _ -> {:error, "Unknown metric '#{inspect(metric_str)}'"} + end + end def parse_filters(filters) when is_list(filters) do parse_list(filters, &parse_filter/1) @@ -284,7 +280,7 @@ defmodule Plausible.Stats.Filters.QueryParser do end end - defp preload_goals_if_needed(site, filters, dimensions) do + def preload_goals_if_needed(site, filters, dimensions) do goal_filters? = Enum.any?(filters, fn [_, filter_key | _rest] -> filter_key == "event:goal" end) diff --git a/lib/plausible/stats/filters/where_builder.ex b/lib/plausible/stats/filters/where_builder.ex index e04396d99..257d4257a 100644 --- a/lib/plausible/stats/filters/where_builder.ex +++ b/lib/plausible/stats/filters/where_builder.ex @@ -10,6 +10,8 @@ defmodule Plausible.Stats.Filters.WhereBuilder do use Plausible.Stats.Fragments + require Logger + @sessions_only_visit_fields [ :entry_page, :exit_page, @@ -150,6 +152,15 @@ defmodule Plausible.Stats.Filters.WhereBuilder do true end + defp add_filter(table, _query, filter) do + Logger.info("Unable to process garbage filter. No results are returned", + table: table, + filter: filter + ) + + false + end + defp filter_custom_prop(prop_name, column_name, [:is, _, values]) do none_value_included = Enum.member?(values, "(none)") diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.ex index a446817a3..5b774d9ea 100644 --- a/lib/plausible/stats/imported/base.ex +++ b/lib/plausible/stats/imported/base.ex @@ -54,10 +54,12 @@ defmodule Plausible.Stats.Imported.Base do def property_to_table_mappings(), do: @property_to_table_mappings def query_imported(site, query) do - query - |> transform_filters() - |> decide_table() - |> query_imported(site, query) + [table] = + query + |> transform_filters() + |> decide_tables() + + query_imported(table, site, query) end def query_imported(table, site, query) do @@ -75,13 +77,13 @@ defmodule Plausible.Stats.Imported.Base do |> apply_filter(query) end - def decide_table(query) do + def decide_tables(query) do query = transform_filters(query) if custom_prop_query?(query) do do_decide_custom_prop_table(query) else - do_decide_table(query) + do_decide_tables(query) end end @@ -92,19 +94,17 @@ defmodule Plausible.Stats.Imported.Base do [:is, "event:name", ["pageview"]] -> true _ -> false end) - |> Enum.flat_map(fn filter -> - case filter do - [op, "event:goal", events] -> - events - |> Enum.group_by(&elem(&1, 0), &elem(&1, 1)) - |> Enum.map(fn - {:event, names} -> [op, "event:name", names] - {:page, pages} -> [op, "event:page", pages] - end) + |> Enum.flat_map(fn + [op, "event:goal", clauses] -> + clauses + |> Enum.group_by(&elem(&1, 0), &elem(&1, 1)) + |> Enum.map(fn + {:event, names} -> [op, "event:name", names] + {:page, pages} -> [op, "event:page", pages] + end) - filter -> - [filter] - end + filter -> + [filter] end) struct!(query, filters: new_filters) @@ -136,10 +136,10 @@ defmodule Plausible.Stats.Imported.Base do do_decide_custom_prop_table(query, custom_prop_filter) _ -> - nil + [] end else - nil + [] end end @@ -158,23 +158,27 @@ defmodule Plausible.Stats.Imported.Base do end) if has_required_name_filter? and not has_unsupported_filters? do - "imported_custom_events" + ["imported_custom_events"] else - nil + [] end end - defp do_decide_table(%Query{filters: [], dimensions: []}), do: "imported_visitors" + defp do_decide_tables(%Query{filters: [], dimensions: []}), do: ["imported_visitors"] - defp do_decide_table(%Query{filters: [], dimensions: ["event:goal"]}) do - "imported_custom_events" + defp do_decide_tables(%Query{filters: [], dimensions: ["event:goal"]}) do + ["imported_pages", "imported_custom_events"] end - defp do_decide_table(%Query{filters: [], dimensions: [dimension]}) do - @property_to_table_mappings[dimension] + defp do_decide_tables(%Query{filters: [], dimensions: [dimension]}) do + if Map.has_key?(@property_to_table_mappings, dimension) do + [@property_to_table_mappings[dimension]] + else + [] + end end - defp do_decide_table(%Query{filters: filters, dimensions: ["event:goal"]}) do + defp do_decide_tables(%Query{filters: filters, dimensions: ["event:goal"]}) do filter_props = Enum.map(filters, &Enum.at(&1, 1)) any_event_name_filters? = "event:name" in filter_props @@ -182,27 +186,28 @@ defmodule Plausible.Stats.Imported.Base do any_other_filters? = Enum.any?(filter_props, &(&1 not in ["event:page", "event:name"])) cond do - any_other_filters? -> nil - any_event_name_filters? and not any_page_filters? -> "imported_custom_events" - any_page_filters? and not any_event_name_filters? -> "imported_pages" - true -> nil + any_other_filters? -> [] + any_event_name_filters? and not any_page_filters? -> ["imported_custom_events"] + any_page_filters? and not any_event_name_filters? -> ["imported_pages"] + true -> [] end end - defp do_decide_table(%Query{filters: filters, dimensions: dimensions}) do + defp do_decide_tables(%Query{filters: filters, dimensions: dimensions}) do table_candidates = filters |> Enum.map(fn [_, filter_key | _] -> filter_key end) |> Enum.concat(dimensions) |> Enum.map(fn "visit:screen" -> "visit:device" - prop -> prop + dimension -> dimension end) |> Enum.map(&@property_to_table_mappings[&1]) case Enum.uniq(table_candidates) do - [candidate] -> candidate - _ -> nil + [nil] -> [] + [candidate] -> [candidate] + _ -> [] end end diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.ex index 6542ab44f..57cd6737b 100644 --- a/lib/plausible/stats/imported/imported.ex +++ b/lib/plausible/stats/imported/imported.ex @@ -4,10 +4,11 @@ defmodule Plausible.Stats.Imported do import Ecto.Query import Plausible.Stats.Fragments + import Plausible.Stats.Util, only: [shortname: 2] - alias Plausible.Stats.Base alias Plausible.Stats.Imported alias Plausible.Stats.Query + alias Plausible.Stats.SQL.QueryBuilder @no_ref "Direct / None" @not_set "(not set)" @@ -15,7 +16,7 @@ defmodule Plausible.Stats.Imported do @property_to_table_mappings Imported.Base.property_to_table_mappings() - @imported_properties Map.keys(@property_to_table_mappings) ++ + @imported_dimensions Map.keys(@property_to_table_mappings) ++ Plausible.Imported.imported_custom_props() @goals_with_url Plausible.Imported.goals_with_url() @@ -37,7 +38,7 @@ defmodule Plausible.Stats.Imported do (see `@goals_with_url` and `@goals_with_path`). """ def schema_supports_query?(query) do - not is_nil(Imported.Base.decide_table(query)) + length(Imported.Base.decide_tables(query)) > 0 end def merge_imported_country_suggestions(native_q, _site, %Plausible.Stats.Query{ @@ -267,78 +268,6 @@ defmodule Plausible.Stats.Imported do def merge_imported(q, _, %Query{include_imported: false}, _), do: q - # Note: Only called for APIv2, old APIs use merge_imported_pageview_goals - def merge_imported(q, site, %Query{dimensions: ["event:goal"]} = query, metrics) - when query.v2 do - {events, page_regexes} = Filters.Utils.split_goals_query_expressions(query.preloaded_goals) - - events_q = - "imported_custom_events" - |> Imported.Base.query_imported(site, query) - |> where([i], i.visitors > 0) - |> select_merge([i], %{ - dim0: selected_as(fragment("-indexOf(?, ?)", ^events, i.name), :dim0) - }) - |> select_imported_metrics(metrics) - |> group_by([], selected_as(:dim0)) - |> where([], selected_as(:dim0) != 0) - - pages_q = - "imported_pages" - |> Imported.Base.query_imported(site, query) - |> where([i], i.visitors > 0) - |> where( - [i], - fragment("notEmpty(multiMatchAllIndices(?, ?) as indices)", i.page, ^page_regexes) - ) - |> join(:array, index in fragment("indices")) - |> group_by([_i, index], index) - |> select_merge([_i, index], %{ - dim0: type(fragment("?", index), :integer) - }) - |> select_imported_metrics(metrics) - - q - |> naive_dimension_join(events_q, metrics) - |> naive_dimension_join(pages_q, metrics) - end - - def merge_imported(q, site, %Query{dimensions: [dimension]} = query, metrics) - when dimension in @imported_properties do - dim = Plausible.Stats.Filters.without_prefix(dimension) - - imported_q = - site - |> Imported.Base.query_imported(query) - |> where([i], i.visitors > 0) - |> group_imported_by(dim, query) - |> select_imported_metrics(metrics) - - join_on = - case dim do - _ when dim in [:url, :path] and not query.v2 -> - dynamic([s, i], s.breakdown_prop_value == i.breakdown_prop_value) - - :os_version when not query.v2 -> - dynamic([s, i], s.os == i.os and s.os_version == i.os_version) - - :browser_version when not query.v2 -> - dynamic([s, i], s.browser == i.browser and s.browser_version == i.browser_version) - - dim -> - dynamic([s, i], field(s, ^shortname(query, dim)) == field(i, ^shortname(query, dim))) - end - - from(s in Ecto.Query.subquery(q), - full_join: i in subquery(imported_q), - on: ^join_on, - select: %{} - ) - |> select_joined_dimension(dim, query) - |> select_joined_metrics(metrics) - |> apply_order_by(query, metrics) - end - def merge_imported(q, site, %Query{dimensions: []} = query, metrics) do imported_q = site @@ -353,51 +282,82 @@ defmodule Plausible.Stats.Imported do |> select_joined_metrics(metrics) end - def merge_imported(q, _, _, _), do: q + def merge_imported(q, site, %Query{dimensions: ["event:goal"]} = query, metrics) do + {events, page_regexes} = Filters.Utils.split_goals_query_expressions(query.preloaded_goals) - def merge_imported_pageview_goals(q, _, %Query{include_imported: false}, _, _), do: q + Imported.Base.decide_tables(query) + |> Enum.map(fn + "imported_custom_events" -> + Imported.Base.query_imported("imported_custom_events", site, query) + |> where([i], i.visitors > 0) + |> select_merge([i], %{ + dim0: + selected_as( + fragment("-indexOf(?, ?)", type(^events, {:array, :string}), i.name), + :dim0 + ) + }) + |> select_imported_metrics(metrics) + |> group_by([], selected_as(:dim0)) + |> where([], selected_as(:dim0) != 0) - def merge_imported_pageview_goals(q, site, query, page_exprs, metrics) do - if Imported.Base.decide_table(query) == "imported_pages" do - page_regexes = Enum.map(page_exprs, &Base.page_regex/1) - - imported_q = - "imported_pages" - |> Imported.Base.query_imported(site, query) + "imported_pages" -> + Imported.Base.query_imported("imported_pages", site, query) |> where([i], i.visitors > 0) |> where( [i], - fragment("notEmpty(multiMatchAllIndices(?, ?) as indices)", i.page, ^page_regexes) + fragment( + "notEmpty(multiMatchAllIndices(?, ?) as indices)", + i.page, + type(^page_regexes, {:array, :string}) + ) ) |> join(:array, index in fragment("indices")) |> group_by([_i, index], index) |> select_merge([_i, index], %{ - name: fragment("concat('Visit ', ?[?])", ^page_exprs, index) + dim0: selected_as(type(fragment("?", index), :integer), :dim0) }) |> select_imported_metrics(metrics) + end) + |> Enum.reduce(q, fn imports_q, q -> + naive_dimension_join(q, imports_q, metrics) + end) + end - from(s in Ecto.Query.subquery(q), + def merge_imported(q, site, %Query{dimensions: dimensions} = query, metrics) do + if merge_imported_dimensions?(dimensions) do + imported_q = + site + |> Imported.Base.query_imported(query) + |> where([i], i.visitors > 0) + |> group_imported_by(query) + |> select_imported_metrics(metrics) + + from(s in subquery(q), full_join: i in subquery(imported_q), - on: s.name == i.name, + on: ^QueryBuilder.build_group_by_join(query), select: %{} ) - |> select_joined_dimension(:name, query) + |> select_joined_dimensions(query) |> select_joined_metrics(metrics) else q end end + def merge_imported(q, _, _, _), do: q + + defp merge_imported_dimensions?(dimensions) do + dimensions in [["visit:browser", "visit:browser_version"], ["visit:os", "visit:os_version"]] or + (length(dimensions) == 1 and hd(dimensions) in @imported_dimensions) + end + def total_imported_visitors(site, query) do site |> Imported.Base.query_imported(query) |> select_merge([i], %{total_visitors: fragment("sum(?)", i.visitors)}) end - # :TRICKY: Handle backwards compatibility with old breakdown module - defp shortname(query, _dim) when query.v2, do: :dim0 - defp shortname(_query, dim), do: dim - defp select_imported_metrics(q, []), do: q defp select_imported_metrics(q, [:visitors | rest]) do @@ -592,190 +552,175 @@ defmodule Plausible.Stats.Imported do |> select_imported_metrics(rest) end - defp group_imported_by(q, dim, query) when dim in [:source, :referrer] do + defp group_imported_by(q, query) do + Enum.reduce(query.dimensions, q, fn dimension, q -> + dim = Plausible.Stats.Filters.without_prefix(dimension) + + group_imported_by(q, dim, shortname(query, dimension)) + end) + end + + defp group_imported_by(q, dim, key) when dim in [:source, :referrer] do q |> group_by([i], field(i, ^dim)) |> select_merge([i], %{ - ^shortname(query, dim) => - fragment( - "if(empty(?), ?, ?)", - field(i, ^dim), - @no_ref, - field(i, ^dim) + ^key => + selected_as( + fragment( + "if(empty(?), ?, ?)", + field(i, ^dim), + @no_ref, + field(i, ^dim) + ), + ^key ) }) end - defp group_imported_by(q, dim, query) + defp group_imported_by(q, dim, key) when dim in [:utm_source, :utm_medium, :utm_campaign, :utm_term, :utm_content] do q |> group_by([i], field(i, ^dim)) |> where([i], fragment("not empty(?)", field(i, ^dim))) - |> select_merge([i], %{^shortname(query, dim) => field(i, ^dim)}) + |> select_merge([i], %{^key => selected_as(field(i, ^dim), ^key)}) end - defp group_imported_by(q, :page, query) do + defp group_imported_by(q, :page, key) do q |> group_by([i], i.page) - |> select_merge([i], %{^shortname(query, :page) => i.page, time_on_page: sum(i.time_on_page)}) + |> select_merge([i], %{^key => selected_as(i.page, ^key), time_on_page: sum(i.time_on_page)}) end - defp group_imported_by(q, :country, query) do + defp group_imported_by(q, :country, key) do q |> group_by([i], i.country) |> where([i], i.country != "ZZ") - |> select_merge([i], %{^shortname(query, :country) => i.country}) + |> select_merge([i], %{^key => selected_as(i.country, ^key)}) end - defp group_imported_by(q, :region, query) do + defp group_imported_by(q, :region, key) do q |> group_by([i], i.region) |> where([i], i.region != "") - |> select_merge([i], %{^shortname(query, :region) => i.region}) + |> select_merge([i], %{^key => selected_as(i.region, ^key)}) end - defp group_imported_by(q, :city, query) do + defp group_imported_by(q, :city, key) do q |> group_by([i], i.city) |> where([i], i.city != 0 and not is_nil(i.city)) - |> select_merge([i], %{^shortname(query, :city) => i.city}) + |> select_merge([i], %{^key => selected_as(i.city, ^key)}) end - defp group_imported_by(q, dim, query) when dim in [:device, :browser] do + defp group_imported_by(q, dim, key) when dim in [:device, :browser] do q |> group_by([i], field(i, ^dim)) |> select_merge([i], %{ - ^shortname(query, dim) => - fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim)) - }) - end - - defp group_imported_by(q, :browser_version, query) do - q - |> group_by([i], [i.browser, i.browser_version]) - |> select_merge([i], %{ - ^shortname(query, :browser) => - fragment("if(empty(?), ?, ?)", i.browser, @not_set, i.browser), - ^shortname(query, :browser_version) => - fragment( - "if(empty(?), ?, ?)", - i.browser_version, - @not_set, - i.browser_version + ^key => + selected_as( + fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim)), + ^key ) }) end - defp group_imported_by(q, :os, query) do + defp group_imported_by(q, :browser_version, key) do + q + |> group_by([i], [i.browser_version]) + |> select_merge([i], %{ + ^key => + selected_as( + fragment( + "if(empty(?), ?, ?)", + i.browser_version, + @not_set, + i.browser_version + ), + ^key + ) + }) + end + + defp group_imported_by(q, :os, key) do q |> group_by([i], i.operating_system) |> select_merge([i], %{ - ^shortname(query, :os) => - fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system) - }) - end - - defp group_imported_by(q, :os_version, query) do - q - |> group_by([i], [i.operating_system, i.operating_system_version]) - |> select_merge([i], %{ - ^shortname(query, :os) => - fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system), - ^shortname(query, :os_version) => - fragment( - "if(empty(?), ?, ?)", - i.operating_system_version, - @not_set, - i.operating_system_version + ^key => + selected_as( + fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system), + ^key ) }) end - defp group_imported_by(q, dim, query) when dim in [:entry_page, :exit_page] do + defp group_imported_by(q, :os_version, key) do + q + |> group_by([i], [i.operating_system_version]) + |> select_merge([i], %{ + ^key => + selected_as( + fragment( + "if(empty(?), ?, ?)", + i.operating_system_version, + @not_set, + i.operating_system_version + ), + ^key + ) + }) + end + + defp group_imported_by(q, dim, key) when dim in [:entry_page, :exit_page] do q |> group_by([i], field(i, ^dim)) - |> select_merge([i], %{^shortname(query, dim) => field(i, ^dim)}) + |> select_merge([i], %{^key => selected_as(field(i, ^dim), ^key)}) end - defp group_imported_by(q, :name, query) do + defp group_imported_by(q, :name, key) do q |> group_by([i], i.name) - |> select_merge([i], %{^shortname(query, :name) => i.name}) + |> select_merge([i], %{^key => selected_as(i.name, ^key)}) end - defp group_imported_by(q, :url, query) when query.v2 do + defp group_imported_by(q, :url, key) do q |> group_by([i], i.link_url) |> select_merge([i], %{ - ^shortname(query, :url) => fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none) + ^key => selected_as(fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none), ^key) }) end - defp group_imported_by(q, :url, _query) do - q - |> group_by([i], i.link_url) - |> select_merge([i], %{ - breakdown_prop_value: fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none) - }) - end - - defp group_imported_by(q, :path, query) when query.v2 do + defp group_imported_by(q, :path, key) do q |> group_by([i], i.path) |> select_merge([i], %{ - ^shortname(query, :path) => fragment("if(not empty(?), ?, ?)", i.path, i.path, @none) + ^key => selected_as(fragment("if(not empty(?), ?, ?)", i.path, i.path, @none), ^key) }) end - defp group_imported_by(q, :path, _query) do - q - |> group_by([i], i.path) - |> select_merge([i], %{ - breakdown_prop_value: fragment("if(not empty(?), ?, ?)", i.path, i.path, @none) - }) + defp select_joined_dimensions(q, query) do + Enum.reduce(query.dimensions, q, fn dimension, q -> + select_joined_dimension(q, dimension, shortname(query, dimension)) + end) end - defp select_joined_dimension(q, :city, query) do + defp select_joined_dimension(q, "visit:city", key) do select_merge(q, [s, i], %{ - ^shortname(query, :city) => fragment("greatest(?,?)", i.city, s.city) + ^key => selected_as(fragment("greatest(?,?)", field(i, ^key), field(s, ^key)), ^key) }) end - defp select_joined_dimension(q, :os_version, query) when not query.v2 do + defp select_joined_dimension(q, _dimension, key) do select_merge(q, [s, i], %{ - os: fragment("if(empty(?), ?, ?)", s.os, i.os, s.os), - os_version: fragment("if(empty(?), ?, ?)", s.os_version, i.os_version, s.os_version) - }) - end - - defp select_joined_dimension(q, :browser_version, query) when not query.v2 do - select_merge(q, [s, i], %{ - browser: fragment("if(empty(?), ?, ?)", s.browser, i.browser, s.browser), - browser_version: - fragment("if(empty(?), ?, ?)", s.browser_version, i.browser_version, s.browser_version) - }) - end - - defp select_joined_dimension(q, dim, query) when dim in [:url, :path] and not query.v2 do - select_merge(q, [s, i], %{ - breakdown_prop_value: - fragment( - "if(empty(?), ?, ?)", - s.breakdown_prop_value, - i.breakdown_prop_value, - s.breakdown_prop_value - ) - }) - end - - defp select_joined_dimension(q, dim, query) do - select_merge(q, [s, i], %{ - ^shortname(query, dim) => - fragment( - "if(empty(?), ?, ?)", - field(s, ^shortname(query, dim)), - field(i, ^shortname(query, dim)), - field(s, ^shortname(query, dim)) + ^key => + selected_as( + fragment( + "if(empty(?), ?, ?)", + field(s, ^key), + field(i, ^key), + field(s, ^key) + ), + ^key ) }) end @@ -882,21 +827,12 @@ defmodule Plausible.Stats.Imported do |> select_joined_metrics(rest) end - defp apply_order_by(q, %Query{v2: true}, _), do: q - - defp apply_order_by(q, query, [:visitors | rest]) do - order_by(q, [s, i], desc: s.visitors + i.visitors) - |> apply_order_by(query, rest) - end - - defp apply_order_by(q, _query, _), do: q - defp naive_dimension_join(q1, q2, metrics) do - from(a in Ecto.Query.subquery(q1), + from(a in subquery(q1), full_join: b in subquery(q2), on: a.dim0 == b.dim0, select: %{ - dim0: fragment("coalesce(?, ?)", a.dim0, b.dim0) + dim0: selected_as(fragment("if(? != 0, ?, ?)", a.dim0, a.dim0, b.dim0), :dim0) } ) |> select_joined_metrics(metrics) diff --git a/lib/plausible/stats/metrics.ex b/lib/plausible/stats/metrics.ex index 8ed71dd2e..29319d07a 100644 --- a/lib/plausible/stats/metrics.ex +++ b/lib/plausible/stats/metrics.ex @@ -17,7 +17,8 @@ defmodule Plausible.Stats.Metrics do :events, :conversion_rate, :group_conversion_rate, - :time_on_page + :time_on_page, + :percentage ] ++ on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: []) @metric_mappings Enum.into(@all_metrics, %{}, fn metric -> {to_string(metric), metric} end) @@ -27,4 +28,8 @@ defmodule Plausible.Stats.Metrics do def from_string!(str) do Map.fetch!(@metric_mappings, str) end + + def from_string(str) do + Map.fetch(@metric_mappings, str) + end end diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index 86fc13220..75fb8c503 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -18,7 +18,12 @@ defmodule Plausible.Stats.Query do order_by: nil, timezone: nil, v2: false, - preloaded_goals: [] + legacy_breakdown: false, + preloaded_goals: [], + include: %{ + imports: false, + time_labels: false + } require OpenTelemetry.Tracer, as: Tracer alias Plausible.Stats.{Filters, Interval, Imported} @@ -225,6 +230,12 @@ defmodule Plausible.Stats.Query do struct!(query, filters: Filters.parse(params["filters"])) end + def set(query, keywords) do + query + |> struct!(keywords) + |> refresh_imported_opts() + end + @spec set_dimensions(t(), list(String.t())) :: t() def set_dimensions(query, dimensions) do query diff --git a/lib/plausible/stats/query_optimizer.ex b/lib/plausible/stats/query_optimizer.ex index c1c38e5fb..25146d950 100644 --- a/lib/plausible/stats/query_optimizer.ex +++ b/lib/plausible/stats/query_optimizer.ex @@ -3,6 +3,7 @@ defmodule Plausible.Stats.QueryOptimizer do Methods to manipulate Query for business logic reasons before building an ecto query. """ + use Plausible alias Plausible.Stats.{Query, TableDecider, Util} @doc """ @@ -148,8 +149,17 @@ defmodule Plausible.Stats.QueryOptimizer do dimension -> dimension end) - query - |> Query.set_metrics(session_metrics) - |> Query.set_dimensions(dimensions) + filters = + if "event:page" in query.dimensions do + query.filters + |> Enum.map(fn + [op, "event:page" | rest] -> [op, "visit:entry_page" | rest] + filter -> filter + end) + else + query.filters + end + + Query.set(query, filters: filters, metrics: session_metrics, dimensions: dimensions) end end diff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.ex index e74348dae..d3f38a142 100644 --- a/lib/plausible/stats/sql/expression.ex +++ b/lib/plausible/stats/sql/expression.ex @@ -11,84 +11,121 @@ defmodule Plausible.Stats.SQL.Expression do @no_ref "Direct / None" @not_set "(not set)" - defmacrop field_or_blank_value(expr, empty_value) do + defmacrop field_or_blank_value(expr, empty_value, select_alias) do quote do dynamic( [t], - fragment("if(empty(?), ?, ?)", unquote(expr), unquote(empty_value), unquote(expr)) + selected_as( + fragment("if(empty(?), ?, ?)", unquote(expr), unquote(empty_value), unquote(expr)), + ^unquote(select_alias) + ) ) end end - def dimension("time:hour", query) do - dynamic([t], fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone)) - end - - def dimension("time:day", query) do - dynamic([t], fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone)) - end - - def dimension("time:month", query) do - dynamic([t], fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone)) - end - - def dimension("event:name", _query), do: dynamic([t], t.name) - def dimension("event:page", _query), do: dynamic([t], t.pathname) - def dimension("event:hostname", _query), do: dynamic([t], t.hostname) - - def dimension("event:props:" <> property_name, _query) do + def dimension("time:hour", query, select_alias) do dynamic( [t], - fragment( - "if(not empty(?), ?, '(none)')", - get_by_key(t, :meta, ^property_name), - get_by_key(t, :meta, ^property_name) + selected_as( + fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone), + ^select_alias ) ) end - def dimension("visit:entry_page", _query), do: dynamic([t], t.entry_page) - def dimension("visit:exit_page", _query), do: dynamic([t], t.exit_page) + def dimension("time:day", query, select_alias) do + dynamic( + [t], + selected_as( + fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone), + ^select_alias + ) + ) + end - def dimension("visit:utm_medium", _query), - do: field_or_blank_value(t.utm_medium, @not_set) + def dimension("time:month", query, select_alias) do + dynamic( + [t], + selected_as( + fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone), + ^select_alias + ) + ) + end - def dimension("visit:utm_source", _query), - do: field_or_blank_value(t.utm_source, @not_set) + def dimension("event:name", _query, select_alias), + do: dynamic([t], selected_as(t.name, ^select_alias)) - def dimension("visit:utm_campaign", _query), - do: field_or_blank_value(t.utm_campaign, @not_set) + def dimension("event:page", _query, select_alias), + do: dynamic([t], selected_as(t.pathname, ^select_alias)) - def dimension("visit:utm_content", _query), - do: field_or_blank_value(t.utm_content, @not_set) + def dimension("event:hostname", _query, select_alias), + do: dynamic([t], selected_as(t.hostname, ^select_alias)) - def dimension("visit:utm_term", _query), - do: field_or_blank_value(t.utm_term, @not_set) + def dimension("event:props:" <> property_name, _query, select_alias) do + dynamic( + [t], + selected_as( + fragment( + "if(not empty(?), ?, '(none)')", + get_by_key(t, :meta, ^property_name), + get_by_key(t, :meta, ^property_name) + ), + ^select_alias + ) + ) + end - def dimension("visit:source", _query), - do: field_or_blank_value(t.source, @no_ref) + def dimension("visit:entry_page", _query, select_alias), + do: dynamic([t], selected_as(t.entry_page, ^select_alias)) - def dimension("visit:referrer", _query), - do: field_or_blank_value(t.referrer, @no_ref) + def dimension("visit:exit_page", _query, select_alias), + do: dynamic([t], selected_as(t.exit_page, ^select_alias)) - def dimension("visit:device", _query), - do: field_or_blank_value(t.device, @not_set) + def dimension("visit:utm_medium", _query, select_alias), + do: field_or_blank_value(t.utm_medium, @not_set, select_alias) - def dimension("visit:os", _query), do: field_or_blank_value(t.os, @not_set) + def dimension("visit:utm_source", _query, select_alias), + do: field_or_blank_value(t.utm_source, @not_set, select_alias) - def dimension("visit:os_version", _query), - do: field_or_blank_value(t.os_version, @not_set) + def dimension("visit:utm_campaign", _query, select_alias), + do: field_or_blank_value(t.utm_campaign, @not_set, select_alias) - def dimension("visit:browser", _query), - do: field_or_blank_value(t.browser, @not_set) + def dimension("visit:utm_content", _query, select_alias), + do: field_or_blank_value(t.utm_content, @not_set, select_alias) - def dimension("visit:browser_version", _query), - do: field_or_blank_value(t.browser_version, @not_set) + def dimension("visit:utm_term", _query, select_alias), + do: field_or_blank_value(t.utm_term, @not_set, select_alias) - # :TODO: Locations also set extra filters - def dimension("visit:country", _query), do: dynamic([t], t.country) - def dimension("visit:region", _query), do: dynamic([t], t.region) - def dimension("visit:city", _query), do: dynamic([t], t.city) + def dimension("visit:source", _query, select_alias), + do: field_or_blank_value(t.source, @no_ref, select_alias) + + def dimension("visit:referrer", _query, select_alias), + do: field_or_blank_value(t.referrer, @no_ref, select_alias) + + def dimension("visit:device", _query, select_alias), + do: field_or_blank_value(t.device, @not_set, select_alias) + + def dimension("visit:os", _query, select_alias), + do: field_or_blank_value(t.os, @not_set, select_alias) + + def dimension("visit:os_version", _query, select_alias), + do: field_or_blank_value(t.os_version, @not_set, select_alias) + + def dimension("visit:browser", _query, select_alias), + do: field_or_blank_value(t.browser, @not_set, select_alias) + + def dimension("visit:browser_version", _query, select_alias), + do: field_or_blank_value(t.browser_version, @not_set, select_alias) + + def dimension("visit:country", _query, select_alias), + do: dynamic([t], selected_as(t.country, ^select_alias)) + + def dimension("visit:region", _query, select_alias), + do: dynamic([t], selected_as(t.region, ^select_alias)) + + def dimension("visit:city", _query, select_alias), + do: dynamic([t], selected_as(t.city, ^select_alias)) defmacro event_goal_join(events, page_regexes) do quote do diff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.ex index 40db95f28..7e5e49a88 100644 --- a/lib/plausible/stats/sql/query_builder.ex +++ b/lib/plausible/stats/sql/query_builder.ex @@ -7,7 +7,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do import Plausible.Stats.Imported import Plausible.Stats.Util - alias Plausible.Stats.{Base, Query, QueryOptimizer, TableDecider, Filters, Metrics} + alias Plausible.Stats.{Base, Query, QueryOptimizer, TableDecider, Filters} alias Plausible.Stats.SQL.Expression require Plausible.Stats.SQL.Expression @@ -44,6 +44,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do |> merge_imported(site, events_query, events_query.metrics) |> maybe_add_global_conversion_rate(site, events_query) |> maybe_add_group_conversion_rate(site, events_query) + |> Base.add_percentage_metric(site, events_query, events_query.metrics) end defp join_sessions_if_needed(q, site, query) do @@ -84,6 +85,9 @@ defmodule Plausible.Stats.SQL.QueryBuilder do |> join_events_if_needed(site, sessions_query) |> build_group_by(sessions_query) |> merge_imported(site, sessions_query, sessions_query.metrics) + |> maybe_add_global_conversion_rate(site, sessions_query) + |> maybe_add_group_conversion_rate(site, sessions_query) + |> Base.add_percentage_metric(site, sessions_query, sessions_query.metrics) end def join_events_if_needed(q, site, query) do @@ -123,36 +127,23 @@ defmodule Plausible.Stats.SQL.QueryBuilder do ^shortname(query, dimension) => fragment("?", goal) }, group_by: goal, - where: goal != 0 + where: goal != 0 and (e.name == "pageview" or goal < 0) ) end defp dimension_group_by(q, query, dimension) do + key = shortname(query, dimension) + q - |> select_merge(^%{shortname(query, dimension) => Expression.dimension(dimension, query)}) - |> group_by(^Expression.dimension(dimension, query)) + |> select_merge(^%{key => Expression.dimension(dimension, query, key)}) + |> group_by([], selected_as(^key)) end - defp build_order_by(q, query, mode) do - Enum.reduce(query.order_by || [], q, &build_order_by(&2, query, &1, mode)) + defp build_order_by(q, query) do + Enum.reduce(query.order_by || [], q, &build_order_by(&2, query, &1)) end - def build_order_by(q, query, {metric_or_dimension, order_direction}, :inner) do - order_by( - q, - [t], - ^{ - order_direction, - if( - Metrics.metric?(metric_or_dimension), - do: dynamic([], selected_as(^shortname(query, metric_or_dimension))), - else: Expression.dimension(metric_or_dimension, query) - ) - } - ) - end - - def build_order_by(q, query, {metric_or_dimension, order_direction}, :outer) do + def build_order_by(q, query, {metric_or_dimension, order_direction}) do order_by( q, [t], @@ -262,10 +253,10 @@ defmodule Plausible.Stats.SQL.QueryBuilder do defp join_query_results({nil, _}, {nil, _}), do: nil defp join_query_results({events_q, events_query}, {nil, _}), - do: events_q |> build_order_by(events_query, :inner) + do: events_q |> build_order_by(events_query) - defp join_query_results({nil, _}, {sessions_q, sessions_query}), - do: sessions_q |> build_order_by(sessions_query, :inner) + defp join_query_results({nil, events_query}, {sessions_q, _}), + do: sessions_q |> build_order_by(events_query) defp join_query_results({events_q, events_query}, {sessions_q, sessions_query}) do join(subquery(events_q), :left, [e], s in subquery(sessions_q), @@ -274,12 +265,12 @@ defmodule Plausible.Stats.SQL.QueryBuilder do |> select_join_fields(events_query, events_query.dimensions, e) |> select_join_fields(events_query, events_query.metrics, e) |> select_join_fields(sessions_query, List.delete(sessions_query.metrics, :sample_percent), s) - |> build_order_by(events_query, :outer) + |> build_order_by(events_query) end - defp build_group_by_join(%Query{dimensions: []}), do: true + def build_group_by_join(%Query{dimensions: []}), do: true - defp build_group_by_join(query) do + def build_group_by_join(query) do query.dimensions |> Enum.map(fn dim -> dynamic([e, s], field(e, ^shortname(query, dim)) == field(s, ^shortname(query, dim))) diff --git a/lib/plausible/stats/table_decider.ex b/lib/plausible/stats/table_decider.ex index c0176d417..7f3efcf25 100644 --- a/lib/plausible/stats/table_decider.ex +++ b/lib/plausible/stats/table_decider.ex @@ -29,34 +29,45 @@ defmodule Plausible.Stats.TableDecider do |> filter_keys() |> partition(query, &filters_partitioner/2) + %{event: event_only_dimensions, session: session_only_dimensions} = + partition(query.dimensions, query, &filters_partitioner/2) + cond do # Only one table needs to be queried - empty?(event_only_metrics) && empty?(event_only_filters) -> + empty?(event_only_metrics) && empty?(event_only_filters) && empty?(event_only_dimensions) -> {[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics} - empty?(session_only_metrics) && empty?(session_only_filters) -> + empty?(session_only_metrics) && empty?(session_only_filters) && + empty?(session_only_dimensions) -> {event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics} - # Filters on both events and sessions, but only one kind of metric - empty?(event_only_metrics) -> + # Filters and/or dimensions on both events and sessions, but only one kind of metric + empty?(event_only_metrics) && empty?(event_only_dimensions) -> {[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics} - empty?(session_only_metrics) -> + empty?(session_only_metrics) && empty?(session_only_dimensions) -> {event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics} - # Default: prefer sessions + # Default: prefer events true -> - {event_only_metrics ++ sample_percent, - session_only_metrics ++ either_metrics ++ sample_percent, other_metrics} + {event_only_metrics ++ either_metrics ++ sample_percent, + session_only_metrics ++ sample_percent, other_metrics} end end defp filter_keys(query) do query.filters |> Enum.map(fn [_, filter_key | _rest] -> filter_key end) - |> Enum.concat(query.dimensions) end + defp metric_partitioner(%Query{v2: true}, :conversion_rate), do: :either + defp metric_partitioner(%Query{v2: true}, :group_conversion_rate), do: :either + defp metric_partitioner(%Query{v2: true}, :visitors), do: :either + defp metric_partitioner(%Query{v2: true}, :visits), do: :either + # Note: This is inaccurate when filtering but required for old backwards compatibility + defp metric_partitioner(%Query{legacy_breakdown: true}, :pageviews), do: :either + defp metric_partitioner(%Query{legacy_breakdown: true}, :events), do: :either + defp metric_partitioner(_, :conversion_rate), do: :event defp metric_partitioner(_, :group_conversion_rate), do: :event defp metric_partitioner(_, :average_revenue), do: :event @@ -76,7 +87,7 @@ defmodule Plausible.Stats.TableDecider do # Calculated metrics - handled on callsite separately from other metrics. defp metric_partitioner(_, :time_on_page), do: :other defp metric_partitioner(_, :total_visitors), do: :other - defp metric_partitioner(_, :percentage), do: :other + defp metric_partitioner(_, :percentage), do: :either # Sample percentage is included in both tables if queried. defp metric_partitioner(_, :sample_percent), do: :sample_percent diff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exs index 18087bddf..3bb6f8ca0 100644 --- a/test/plausible/imported/csv_importer_test.exs +++ b/test/plausible/imported/csv_importer_test.exs @@ -622,7 +622,7 @@ defmodule Plausible.Imported.CSVImporterTest do case params_or_site do %Plausible.Site{} = site -> common_params.(site) - |> Map.put("metrics", "visitors,visits,pageviews,visit_duration,bounce_rate") + |> Map.put("metrics", "visitors,visits,visit_duration,bounce_rate") |> Map.put("limit", 1000) |> Map.put("property", by) @@ -669,7 +669,7 @@ defmodule Plausible.Imported.CSVImporterTest do assert exported["pageviews"] == imported["pageviews"] assert exported["bounce_rate"] == imported["bounce_rate"] assert_in_delta exported["visitors"], imported["visitors"], 1 - assert exported["visits"] == imported["visits"] + assert_in_delta exported["visits"], imported["visits"], 1 assert_in_delta exported["visit_duration"], imported["visit_duration"], 1 end) @@ -810,7 +810,7 @@ defmodule Plausible.Imported.CSVImporterTest do _no_diff = 0 end end) - ) == [0.0, 0.0, 0.0, 0.0, 0.03614457831325302] + ) == [0.0, 0.0, 0.0, 0.0, 0.0] # NOTE: city breakdown's visit duration difference is up to 14% assert summary(field(exported_cities, "visit_duration")) == [0, 0.0, 0.0, 1.0, 1718] @@ -829,7 +829,7 @@ defmodule Plausible.Imported.CSVImporterTest do _no_diff = 0 end end) - ) == [0, 0.0, 0.0, 0.0, 0.1428571428571429] + ) == [0, 0.0, 0.0, 0.0, 0.0] # NOTE: city breakdown's visitors relative difference is up to 27% assert summary(field(exported_cities, "visitors")) == [1, 1.0, 1.0, 2.0, 22] diff --git a/test/plausible/imported/google_analytics4_test.exs b/test/plausible/imported/google_analytics4_test.exs index 7253877c4..728378787 100644 --- a/test/plausible/imported/google_analytics4_test.exs +++ b/test/plausible/imported/google_analytics4_test.exs @@ -117,7 +117,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do breakdown_params = common_params - |> Map.put("metrics", "visitors,visits,pageviews,visit_duration,bounce_rate") + |> Map.put("metrics", "visitors,visits,visit_duration,bounce_rate") |> Map.put("limit", 1000) %{key: api_key} = insert(:api_key, user: user) @@ -467,7 +467,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do assert List.first(results) == %{ "bounce_rate" => 35.0, - "pageviews" => 6229, "visit_duration" => 40.0, "visitors" => 4671, "visits" => 4917, @@ -475,12 +474,11 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do } assert List.last(results) == %{ - "bounce_rate" => 100.0, - "pageviews" => 1, - "visit_duration" => 0.0, + "bounce_rate" => 0.0, + "source" => "yahoo", + "visit_duration" => 41.0, "visitors" => 1, - "visits" => 1, - "source" => "petalsearch.com" + "visits" => 1 } end @@ -493,7 +491,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do assert [ %{ "bounce_rate" => 35.0, - "pageviews" => 6399, "utm_medium" => "organic", "visit_duration" => 40.0, "visitors" => 4787, @@ -501,7 +498,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do }, %{ "bounce_rate" => 58.0, - "pageviews" => 491, "utm_medium" => "referral", "visit_duration" => 27.0, "visitors" => 294, @@ -520,7 +516,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do assert List.first(results) == %{ "bounce_rate" => 35.0, - "pageviews" => 838, "visit_duration" => 43.0, "visitors" => 675, "visits" => 712, @@ -529,11 +524,10 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do assert List.last(results) == %{ "bounce_rate" => 0.0, - "pageviews" => 1, - "visit_duration" => 27.0, + "entry_page" => "/znamenitosti-rima-koje-treba-vidjeti", + "visit_duration" => 40.0, "visitors" => 1, - "visits" => 1, - "entry_page" => "/kad-lisce-pada" + "visits" => 1 } end @@ -543,12 +537,11 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do %{"results" => results} = get(conn, "/api/v1/stats/breakdown", params) |> json_response(200) - assert length(results) == 488 + assert length(results) == 494 assert List.first(results) == %{ "bounce_rate" => 35.0, "city" => 792_680, - "pageviews" => 1650, "visit_duration" => 39.0, "visitors" => 1233, "visits" => 1273 @@ -556,9 +549,8 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do assert List.last(results) == %{ "bounce_rate" => 0.0, - "city" => 4_399_605, - "pageviews" => 7, - "visit_duration" => 128.0, + "city" => 11_951_298, + "visit_duration" => 271.0, "visitors" => 1, "visits" => 1 } @@ -574,7 +566,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do assert List.first(results) == %{ "bounce_rate" => 38.0, - "pageviews" => 7041, "visit_duration" => 37.0, "visitors" => 5277, "visits" => 5532, @@ -583,7 +574,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do assert List.last(results) == %{ "bounce_rate" => 37.0, - "pageviews" => 143, "visit_duration" => 60.0, "visitors" => 97, "visits" => 100, @@ -601,7 +591,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do assert List.first(results) == %{ "bounce_rate" => 33.0, - "pageviews" => 8143, "visit_duration" => 50.0, "visitors" => 4625, "visits" => 4655, @@ -610,7 +599,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do assert List.last(results) == %{ "bounce_rate" => 0.0, - "pageviews" => 6, "visit_duration" => 0.0, "visitors" => 1, "visits" => 1, @@ -628,7 +616,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do assert List.first(results) == %{ "bounce_rate" => 34.0, - "pageviews" => 5827, "visit_duration" => 41.0, "visitors" => 4319, "visits" => 4495, @@ -637,7 +624,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do assert List.last(results) == %{ "bounce_rate" => 0.0, - "pageviews" => 6, "visit_duration" => 0.0, "visitors" => 1, "visits" => 1, @@ -657,7 +643,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do "bounce_rate" => 32.0, "os" => "Android", "os_version" => "13.0.0", - "pageviews" => 1673, "visit_duration" => 42.0, "visitors" => 1247, "visits" => 1295 @@ -665,17 +650,16 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do assert List.last(results) == %{ "bounce_rate" => 0.0, - "os" => "iOS", - "os_version" => "15.1", - "pageviews" => 1, - "visit_duration" => 54.0, + "os" => "Chrome OS", + "os_version" => "x86_64 15662.76.0", + "visit_duration" => 16.0, "visitors" => 1, "visits" => 1 } end defp assert_pages(conn, params) do - metrics = "visitors,visits,pageviews,time_on_page,visit_duration,bounce_rate" + metrics = "visitors,visits,time_on_page,visit_duration,bounce_rate" params = params @@ -686,7 +670,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do %{"results" => results} = get(conn, "/api/v1/stats/breakdown", params) |> json_response(200) - assert length(results) == 729 + assert length(results) == 730 # The `event:page` breakdown is currently using the `entry_page` # property to allow querying session metrics. @@ -696,7 +680,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do # it will allow us to assert on the session metrics as well. assert Enum.at(results, 2) == %{ "page" => "/", - "pageviews" => 5537, "time_on_page" => 17.677262055264585, "visitors" => 371, "visits" => 212, @@ -707,13 +690,12 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do # This page was never an entry_page in the imported data, and # therefore the session metrics are returned as `nil`. assert List.last(results) == %{ - "page" => "/5-dobrih-razloga-zasto-zapoceti-dan-zobenom-kasom/", - "pageviews" => 2, - "time_on_page" => 10.0, + "bounce_rate" => 0.0, + "page" => "/znamenitosti-rima-koje-treba-vidjeti/", + "time_on_page" => 40.0, + "visit_duration" => 0.0, "visitors" => 1, - "visits" => 1, - "bounce_rate" => nil, - "visit_duration" => nil + "visits" => 1 } end end diff --git a/test/plausible/stats/table_decider_test.exs b/test/plausible/stats/table_decider_test.exs index 7226542c1..b216d02c7 100644 --- a/test/plausible/stats/table_decider_test.exs +++ b/test/plausible/stats/table_decider_test.exs @@ -76,7 +76,7 @@ defmodule Plausible.Stats.TableDeciderTest do query = make_query(false, []) assert partition_metrics([:time_on_page, :percentage, :total_visitors], query) == - {[], [], [:time_on_page, :percentage, :total_visitors]} + {[], [:percentage], [:time_on_page, :total_visitors]} end test "raises if unknown metric" do @@ -108,11 +108,11 @@ defmodule Plausible.Stats.TableDeciderTest do {[], [:visit_duration, :visits], []} end - test "metrics that can be calculated on either are biased to sessions" do + test "metrics that can be calculated on either are biased to events" do query = make_query(true, []) assert partition_metrics([:bounce_rate, :total_revenue, :visitors], query) == - {[:total_revenue], [:bounce_rate, :visitors], []} + {[:total_revenue, :visitors], [:bounce_rate], []} end test "sample_percent is handled with either metrics" do diff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv index 1cef2d8c4..4741db70e 100644 --- a/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv +++ b/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv @@ -1,2 +1,2 @@ name,visitors,pageviews,bounce_rate,time_on_page -/some-other-page,1,1,,60.0 +/some-other-page,1,1,0,60.0 diff --git a/test/plausible_web/controllers/CSVs/30d/pages.csv b/test/plausible_web/controllers/CSVs/30d/pages.csv index e2990caa9..62b246c42 100644 --- a/test/plausible_web/controllers/CSVs/30d/pages.csv +++ b/test/plausible_web/controllers/CSVs/30d/pages.csv @@ -1,4 +1,4 @@ name,visitors,pageviews,bounce_rate,time_on_page /,4,3,67, /signup,1,1,0,60.0 -/some-other-page,1,1,,60.0 +/some-other-page,1,1,0,60.0 diff --git a/test/plausible_web/controllers/CSVs/6m/pages.csv b/test/plausible_web/controllers/CSVs/6m/pages.csv index 805158544..ed895e036 100644 --- a/test/plausible_web/controllers/CSVs/6m/pages.csv +++ b/test/plausible_web/controllers/CSVs/6m/pages.csv @@ -1,4 +1,4 @@ name,visitors,pageviews,bounce_rate,time_on_page /,5,4,75, /signup,1,1,0,60.0 -/some-other-page,1,1,,60.0 +/some-other-page,1,1,0,60.0 diff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs index 8be42e318..d6b9a305b 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs @@ -890,8 +890,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do assert json_response(conn, 200) == %{ "results" => [ %{"page" => "/", "pageviews" => 2}, - %{"page" => "/plausible.io", "pageviews" => 1}, - %{"page" => "/include-me", "pageviews" => 1} + %{"page" => "/include-me", "pageviews" => 1}, + %{"page" => "/plausible.io", "pageviews" => 1} ] } end @@ -1023,7 +1023,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do "period" => "day", "date" => "2021-01-01", "property" => "visit:exit_page", - "metrics" => "visitors,visits,pageviews,bounce_rate,visit_duration,events", + "metrics" => "visitors,visits,bounce_rate,visit_duration,events,pageviews", "with_imported" => "true" }) @@ -1031,21 +1031,21 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do "results" => [ %{ "bounce_rate" => 0.0, - "events" => 7, "exit_page" => "/b", - "pageviews" => 7, "visit_duration" => 150.0, "visitors" => 3, - "visits" => 4 + "visits" => 4, + "events" => 7, + "pageviews" => 7 }, %{ "bounce_rate" => 100.0, - "events" => 1, "exit_page" => "/a", - "pageviews" => 1, "visit_duration" => 0.0, "visitors" => 1, - "visits" => 1 + "visits" => 1, + "events" => 1, + "pageviews" => 1 } ] } @@ -2176,8 +2176,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do assert json_response(conn, 200) == %{ "results" => [ - %{"page" => "/plausible.io", "bounce_rate" => 100}, - %{"page" => "/important-page", "bounce_rate" => 100} + %{"page" => "/important-page", "bounce_rate" => 100}, + %{"page" => "/plausible.io", "bounce_rate" => 100} ] } end @@ -2596,14 +2596,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do assert json_response(conn, 200) == %{ "results" => [ - %{ - "page" => "/B", - "time_on_page" => 90.0 - }, %{ "page" => "/A", "time_on_page" => 60.0 }, + %{ + "page" => "/B", + "time_on_page" => 90.0 + }, %{ "page" => "/C", "time_on_page" => nil @@ -3045,13 +3045,13 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do assert json_response(conn, 200) == %{ "results" => [ - %{ - "entry_page" => "/entry-page-1", - "bounce_rate" => 0 - }, %{ "entry_page" => "/entry-page-2", "bounce_rate" => 100 + }, + %{ + "entry_page" => "/entry-page-1", + "bounce_rate" => 0 } ] } @@ -3146,6 +3146,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do }, %{ "page" => "/plausible.io", + # Breaks for event:page breakdown since visitors is calculated based on entry_page :/ "visitors" => 2, "bounce_rate" => 100, "visit_duration" => 0, @@ -3290,8 +3291,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do assert %{"browser" => "Chrome", "events" => 1} = breakdown_and_first.("visit:browser") assert %{"device" => "Desktop", "events" => 1} = breakdown_and_first.("visit:device") - assert %{"entry_page" => "/test", "events" => 1} = breakdown_and_first.("visit:entry_page") - assert %{"exit_page" => "/test", "events" => 1} = breakdown_and_first.("visit:exit_page") assert %{"country" => "EE", "events" => 1} = breakdown_and_first.("visit:country") assert %{"os" => "Mac", "events" => 1} = breakdown_and_first.("visit:os") assert %{"page" => "/test", "events" => 1} = breakdown_and_first.("event:page") diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs index cc4aec1ba..331687e1f 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs @@ -1477,7 +1477,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do conn = post(conn, "/api/v2/query", %{ "site_id" => site.domain, - "metrics" => ["visitors"], + "metrics" => ["visitors", "percentage"], "date_range" => ["2021-01-01", "2021-01-01"], "dimensions" => [unquote(dimension)] }) @@ -1485,9 +1485,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do %{"results" => results} = json_response(conn, 200) assert results == [ - %{"dimensions" => [unquote(value1)], "metrics" => [3]}, - %{"dimensions" => [unquote(value2)], "metrics" => [2]}, - %{"dimensions" => [unquote(blank_value)], "metrics" => [1]} + %{"dimensions" => [unquote(value1)], "metrics" => [3, 50]}, + %{"dimensions" => [unquote(value2)], "metrics" => [2, 33.3]}, + %{"dimensions" => [unquote(blank_value)], "metrics" => [1, 16.7]} ] end end @@ -3463,6 +3463,48 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do %{"dimensions" => ["Chrome"], "metrics" => [1]} ] end + + test "all metrics for breakdown by event prop", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + user_id: 1, + pathname: "/", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: 1, + pathname: "/plausible.io", + timestamp: ~N[2021-01-01 00:10:00] + ), + build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]), + build(:pageview, + pathname: "/plausible.io", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => [ + "visitors", + "visits", + "pageviews", + "events", + "bounce_rate", + "visit_duration" + ], + "date_range" => "all", + "dimensions" => ["event:page"] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["/"], "metrics" => [2, 2, 2, 2, 50, 300]}, + %{"dimensions" => ["/plausible.io"], "metrics" => [2, 2, 2, 2, 100, 0]} + ] + end end describe "imported data" do @@ -3589,10 +3631,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do assert %{"dimensions" => ["Desktop"], "metrics" => [1]} = breakdown_and_first.("visit:device") - # :TODO: These should not pass validation - not available on events. - # visit dimension and event-only metric - # assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("visit:entry_page") - # assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("visit:exit_page") assert %{"dimensions" => ["EE"], "metrics" => [1]} = breakdown_and_first.("visit:country") assert %{"dimensions" => ["Mac"], "metrics" => [1]} = breakdown_and_first.("visit:os") assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("event:page") @@ -3865,6 +3903,56 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do assert meta["warning"] =~ "Imported stats are not included in the results" end + + test "imported country, region and city data", + %{ + conn: conn, + site: site + } do + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:pageview, + timestamp: ~N[2021-01-01 00:15:00], + country_code: "DE", + subdivision1_code: "DE-BE", + city_geoname_id: 2_950_159 + ), + build(:pageview, + timestamp: ~N[2021-01-01 00:15:00], + country_code: "DE", + subdivision1_code: "DE-BE", + city_geoname_id: 2_950_159 + ), + build(:pageview, + timestamp: ~N[2021-01-01 00:15:00], + country_code: "EE", + subdivision1_code: "EE-37", + city_geoname_id: 588_409 + ), + build(:imported_locations, country: "EE", region: "EE-37", city: 588_409, visitors: 33) + ]) + + for {dimension, stats_value, imports_value} <- [ + {"visit:country", "DE", "EE"}, + {"visit:region", "DE-BE", "EE-37"}, + {"visit:city", 2_950_159, 588_409} + ] do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => [dimension], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => [imports_value], "metrics" => [34]}, + %{"dimensions" => [stats_value], "metrics" => [2]} + ] + end + end end test "multiple breakdown timeseries with sources", %{conn: conn, site: site} do diff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs index 24e18ed2d..a5bcdb38b 100644 --- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs @@ -207,6 +207,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do ] end + @tag capture_log: true test "garbage filters don't crash the call", %{conn: conn, site: site} do filters = "{\"source\":\"Direct / None\",\"screen\":\"Desktop\",\"browser\":\"Chrome\",\"os\":\"Mac\",\"os_version\":\"10.15\",\"country\":\"DE\",\"city\":\"2950159\"}%' AND 2*3*8=6*8 AND 'L9sv'!='L9sv%" diff --git a/test/plausible_web/controllers/api/stats_controller/imported_test.exs b/test/plausible_web/controllers/api/stats_controller/imported_test.exs index 370eb0ecf..16e7a35f9 100644 --- a/test/plausible_web/controllers/api/stats_controller/imported_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/imported_test.exs @@ -601,17 +601,17 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do ) assert json_response(conn, 200)["results"] == [ - %{ - "name" => "blog", - "visitors" => 2, - "bounce_rate" => 50.0, - "visit_duration" => 50.0 - }, %{ "name" => "ad", "visitors" => 2, "bounce_rate" => 100.0, "visit_duration" => 50.0 + }, + %{ + "name" => "blog", + "visitors" => 2, + "bounce_rate" => 50.0, + "visit_duration" => 50.0 } ] end @@ -708,7 +708,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do assert json_response(conn, 200)["results"] == [ %{ - "bounce_rate" => nil, + "bounce_rate" => 0, "time_on_page" => 60, "visitors" => 3, "pageviews" => 4, diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs index 4af5c5cd0..6a55d0bf8 100644 --- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs @@ -340,7 +340,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do "name" => "/blog/other-post", "visitors" => 1, "pageviews" => 1, - "bounce_rate" => nil, + "bounce_rate" => 0, "time_on_page" => nil } ] @@ -392,7 +392,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do "name" => "/blog/other-post", "visitors" => 1, "pageviews" => 1, - "bounce_rate" => nil, + "bounce_rate" => 0, "time_on_page" => nil } ] @@ -744,7 +744,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do "name" => "/blog/post-2", "visitors" => 1, "pageviews" => 1, - "bounce_rate" => nil, + "bounce_rate" => 0, "time_on_page" => nil } ] @@ -789,7 +789,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do "name" => "/blog/(/post-2", "visitors" => 1, "pageviews" => 1, - "bounce_rate" => nil, + "bounce_rate" => 0, "time_on_page" => nil } ] @@ -842,7 +842,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do "name" => "/about", "visitors" => 1, "pageviews" => 1, - "bounce_rate" => nil, + "bounce_rate" => 0, "time_on_page" => nil } ] @@ -940,7 +940,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do "name" => "/" }, %{ - "bounce_rate" => nil, + "bounce_rate" => 0, "time_on_page" => nil, "visitors" => 1, "pageviews" => 1, @@ -1066,7 +1066,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do "visitors" => 2 }, %{ - "bounce_rate" => nil, + "bounce_rate" => 0, "name" => "/exit-blog", "pageviews" => 1, "time_on_page" => nil, @@ -1192,7 +1192,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do "name" => "/" }, %{ - "bounce_rate" => nil, + "bounce_rate" => 0, "time_on_page" => 60, "visitors" => 2, "pageviews" => 2, diff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exs index a76b1da27..ea7b9c15a 100644 --- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs @@ -453,6 +453,9 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do referrer_source: "DuckDuckGo", referrer: "duckduckgo.com" ), + build(:imported_sources, + source: "DuckDuckGo" + ), build(:imported_sources, source: "DuckDuckGo" ) @@ -467,7 +470,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do conn = get(conn, "/api/stats/#{site.domain}/sources?limit=1&page=2&with_imported=true") assert json_response(conn, 200)["results"] == [ - %{"name" => "DuckDuckGo", "visitors" => 2} + %{"name" => "Google", "visitors" => 2} ] end @@ -590,17 +593,17 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ) assert json_response(conn, 200)["results"] == [ - %{ - "name" => "social", - "visitors" => 1, - "bounce_rate" => 0, - "visit_duration" => 900 - }, %{ "name" => "email", "visitors" => 1, "bounce_rate" => 100, "visit_duration" => 0 + }, + %{ + "name" => "social", + "visitors" => 1, + "bounce_rate" => 0, + "visit_duration" => 900 } ] @@ -611,17 +614,17 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ) assert json_response(conn, 200)["results"] == [ - %{ - "name" => "social", - "visitors" => 2, - "bounce_rate" => 50, - "visit_duration" => 800.0 - }, %{ "name" => "email", "visitors" => 2, "bounce_rate" => 50, "visit_duration" => 50 + }, + %{ + "name" => "social", + "visitors" => 2, + "bounce_rate" => 50, + "visit_duration" => 800.0 } ] end