From a9676546dcb82909e8bd0fad8f36233def27f483 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Tue, 9 Jul 2024 14:31:45 +0300 Subject: [PATCH] APIv2: Cleanup (#4308) * Remove a dead method * Move select_event/session_metrics to within QueryBuilder * Make a method private * Move page_regex util around * Move utc_boundaries helper around * Fixups for utc_boundaries * Add legacy notices * Move Stats.query method around * include_sentry_replay_info consistently * Move filters out of select_group_fields * Collapse conditions under select_group_fields * Shorten some imported methods * Use dimension over dim in group_by * Separate SQL query building from imported.ex * props.ex -> legacy_dimensions.ex * Move some query building out of Query.ex * Remove unneeded method * put_filter -> add_filter * Remove some query setters * Moduledoc * Split out validations and import tests from query_test * Move tests around * Split event:goal tests from query_test * Remove redundant filters * Remove dead code * Split special metrics tests from query_test * Legacy module --- extra/lib/plausible/stats/funnel.ex | 2 +- .../google/search_console/filters.ex | 2 +- lib/plausible/stats.ex | 20 +- lib/plausible/stats/aggregate.ex | 6 + lib/plausible/stats/base.ex | 98 +- lib/plausible/stats/breakdown.ex | 6 + lib/plausible/stats/email_report.ex | 6 +- lib/plausible/stats/filters/utils.ex | 13 +- lib/plausible/stats/imported/imported.ex | 481 +----- lib/plausible/stats/imported/sql/builder.ex | 462 ++++++ .../{props.ex => legacy/legacy_dimensions.ex} | 13 +- .../stats/legacy/legacy_query_builder.ex | 180 +++ lib/plausible/stats/query.ex | 218 +-- lib/plausible/stats/sql/query_builder.ex | 16 +- lib/plausible/stats/sql/where_builder.ex | 3 +- lib/plausible/stats/time.ex | 51 + lib/plausible/stats/timeseries.ex | 6 + .../api/external_stats_controller.ex | 4 +- .../controllers/api/stats_controller.ex | 8 +- .../legacy_dashboard_filter_parser_test.exs | 2 +- test/plausible/stats/query_test.exs | 8 +- .../query_goal_dimension_test.exs | 133 ++ .../query_imported_test.exs | 695 +++++++++ .../query_special_metrics_test.exs | 193 +++ .../external_stats_controller/query_test.exs | 1346 +---------------- .../query_validations_test.exs | 234 +++ 26 files changed, 2094 insertions(+), 2112 deletions(-) create mode 100644 lib/plausible/stats/imported/sql/builder.ex rename lib/plausible/stats/{props.ex => legacy/legacy_dimensions.ex} (58%) create mode 100644 lib/plausible/stats/legacy/legacy_query_builder.ex create mode 100644 test/plausible_web/controllers/api/external_stats_controller/query_goal_dimension_test.exs create mode 100644 test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs create mode 100644 test/plausible_web/controllers/api/external_stats_controller/query_special_metrics_test.exs create mode 100644 test/plausible_web/controllers/api/external_stats_controller/query_validations_test.exs diff --git a/extra/lib/plausible/stats/funnel.ex b/extra/lib/plausible/stats/funnel.ex index 121a699fa..a365f3610 100644 --- a/extra/lib/plausible/stats/funnel.ex +++ b/extra/lib/plausible/stats/funnel.ex @@ -110,7 +110,7 @@ defmodule Plausible.Stats.Funnel do %Plausible.Goal{page_path: pathname} when is_binary(pathname) -> if String.contains?(pathname, "*") do - regex = Plausible.Stats.Base.page_regex(pathname) + regex = Plausible.Stats.Filters.Utils.page_regex(pathname) dynamic([], fragment("match(pathname, ?)", ^regex)) else dynamic([], fragment("pathname = ?", ^pathname)) diff --git a/lib/plausible/google/search_console/filters.ex b/lib/plausible/google/search_console/filters.ex index ee9bc2157..43e10e1c8 100644 --- a/lib/plausible/google/search_console/filters.ex +++ b/lib/plausible/google/search_console/filters.ex @@ -1,6 +1,6 @@ defmodule Plausible.Google.SearchConsole.Filters do @moduledoc false - import Plausible.Stats.Base, only: [page_regex: 1] + import Plausible.Stats.Filters.Utils, only: [page_regex: 1] def transform(property, plausible_filters) do search_console_filters = diff --git a/lib/plausible/stats.ex b/lib/plausible/stats.ex index ffa59389e..f22f1b069 100644 --- a/lib/plausible/stats.ex +++ b/lib/plausible/stats.ex @@ -15,6 +15,17 @@ defmodule Plausible.Stats do use Plausible.DebugReplayInfo + def query(site, query) do + include_sentry_replay_info() + + optimized_query = QueryOptimizer.optimize(query) + + optimized_query + |> SQL.QueryBuilder.build(site) + |> ClickhouseRepo.all() + |> QueryResult.from(optimized_query) + end + def breakdown(site, query, metrics, pagination) do include_sentry_replay_info() Breakdown.breakdown(site, query, metrics, pagination) @@ -35,15 +46,6 @@ defmodule Plausible.Stats do CurrentVisitors.current_visitors(site) end - def query(site, query) do - optimized_query = QueryOptimizer.optimize(query) - - optimized_query - |> SQL.QueryBuilder.build(site) - |> ClickhouseRepo.all() - |> QueryResult.from(optimized_query) - end - on_ee do def funnel(site, query, funnel) do include_sentry_replay_info() diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex index 8c5cdc4f7..790b5ed71 100644 --- a/lib/plausible/stats/aggregate.ex +++ b/lib/plausible/stats/aggregate.ex @@ -1,4 +1,10 @@ defmodule Plausible.Stats.Aggregate do + @moduledoc """ + Builds aggregate results for v1 of our stats API and dashboards. + + Avoid adding new logic here - update QueryBuilder etc instead. + """ + use Plausible.ClickhouseRepo use Plausible import Plausible.Stats.Base diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index cb104ffc7..c5304eaac 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -1,10 +1,7 @@ defmodule Plausible.Stats.Base do - use Plausible.ClickhouseRepo use Plausible - use Plausible.Stats.SQL.Fragments - alias Plausible.Stats.{Query, TableDecider, SQL} - alias Plausible.Timezones + alias Plausible.Stats.{TableDecider, SQL} import Ecto.Query def base_event_query(site, query) do @@ -29,7 +26,7 @@ defmodule Plausible.Stats.Base do end end - def query_events(site, query) do + defp query_events(site, query) do q = from(e in "events_v2", where: ^SQL.WhereBuilder.build(:events, site, query)) on_ee do @@ -48,95 +45,4 @@ defmodule Plausible.Stats.Base do q end - - def select_event_metrics(metrics) do - metrics - |> Enum.map(&SQL.Expression.event_metric/1) - |> Enum.reduce(%{}, &Map.merge/2) - end - - def select_session_metrics(metrics, query) do - metrics - |> Enum.map(&SQL.Expression.session_metric(&1, query)) - |> Enum.reduce(%{}, &Map.merge/2) - end - - def filter_converted_sessions(db_query, site, query) do - if Query.has_event_filters?(query) do - converted_sessions = - from(e in query_events(site, query), - select: %{ - session_id: fragment("DISTINCT ?", e.session_id), - _sample_factor: fragment("_sample_factor") - } - ) - - from(s in db_query, - join: cs in subquery(converted_sessions), - on: s.session_id == cs.session_id - ) - else - db_query - end - end - - defp beginning_of_time(candidate, native_stats_start_at) do - if Timex.after?(native_stats_start_at, candidate) do - native_stats_start_at - else - candidate - end - end - - def utc_boundaries(%Query{period: "realtime", now: now}, site) do - last_datetime = - now - |> Timex.shift(seconds: 5) - |> beginning_of_time(site.native_stats_start_at) - |> NaiveDateTime.truncate(:second) - - first_datetime = - now |> Timex.shift(minutes: -5) |> NaiveDateTime.truncate(:second) - - {first_datetime, last_datetime} - end - - def utc_boundaries(%Query{period: "30m", now: now}, site) do - last_datetime = - now - |> Timex.shift(seconds: 5) - |> beginning_of_time(site.native_stats_start_at) - |> NaiveDateTime.truncate(:second) - - first_datetime = - now |> Timex.shift(minutes: -30) |> NaiveDateTime.truncate(:second) - - {first_datetime, last_datetime} - end - - def utc_boundaries(%Query{date_range: date_range}, site) do - {:ok, first} = NaiveDateTime.new(date_range.first, ~T[00:00:00]) - - first_datetime = - first - |> Timezones.to_utc_datetime(site.timezone) - |> beginning_of_time(site.native_stats_start_at) - - {:ok, last} = NaiveDateTime.new(date_range.last |> Timex.shift(days: 1), ~T[00:00:00]) - - last_datetime = Timezones.to_utc_datetime(last, site.timezone) - - {first_datetime, last_datetime} - end - - def page_regex(expr) do - escaped = - expr - |> Regex.escape() - |> String.replace("\\|", "|") - |> String.replace("\\*\\*", ".*") - |> String.replace("\\*", ".*") - - "^#{escaped}$" - end end diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index abf78beac..f4f1d40d2 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -1,4 +1,10 @@ defmodule Plausible.Stats.Breakdown do + @moduledoc """ + Builds breakdown results for v1 of our stats API and dashboards. + + Avoid adding new logic here - update QueryBuilder etc instead. + """ + use Plausible.ClickhouseRepo use Plausible use Plausible.Stats.SQL.Fragments diff --git a/lib/plausible/stats/email_report.ex b/lib/plausible/stats/email_report.ex index 1dc26465b..d745ed797 100644 --- a/lib/plausible/stats/email_report.ex +++ b/lib/plausible/stats/email_report.ex @@ -40,7 +40,7 @@ defmodule Plausible.Stats.EmailReport do end defp put_top_5_pages(stats, site, query) do - query = Query.set_dimensions(query, ["event:page"]) + query = Query.set(query, dimensions: ["event:page"]) pages = Stats.breakdown(site, query, [:visitors], {5, 1}) Map.put(stats, :pages, pages) end @@ -48,8 +48,8 @@ defmodule Plausible.Stats.EmailReport do defp put_top_5_sources(stats, site, query) do query = query - |> Query.put_filter([:is_not, "visit:source", ["Direct / None"]]) - |> Query.set_dimensions(["visit:source"]) + |> Query.add_filter([:is_not, "visit:source", ["Direct / None"]]) + |> Query.set(dimensions: ["visit:source"]) sources = Stats.breakdown(site, query, [:visitors], {5, 1}) diff --git a/lib/plausible/stats/filters/utils.ex b/lib/plausible/stats/filters/utils.ex index 96606c1d0..4869286fa 100644 --- a/lib/plausible/stats/filters/utils.ex +++ b/lib/plausible/stats/filters/utils.ex @@ -76,8 +76,19 @@ defmodule Plausible.Stats.Filters.Utils do events = Enum.map(event_goals, fn {_, event} -> event end) page_regexes = - Enum.map(pageview_goals, fn {_, path} -> Plausible.Stats.Base.page_regex(path) end) + Enum.map(pageview_goals, fn {_, path} -> page_regex(path) end) {events, page_regexes} end + + def page_regex(expr) do + escaped = + expr + |> Regex.escape() + |> String.replace("\\|", "|") + |> String.replace("\\*\\*", ".*") + |> String.replace("\\*", ".*") + + "^#{escaped}$" + end end diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.ex index a1e817e48..ead5a5db8 100644 --- a/lib/plausible/stats/imported/imported.ex +++ b/lib/plausible/stats/imported/imported.ex @@ -3,17 +3,13 @@ defmodule Plausible.Stats.Imported do use Plausible.Stats.SQL.Fragments import Ecto.Query - import Plausible.Stats.Util, only: [shortname: 2] + import Plausible.Stats.Imported.SQL.Builder alias Plausible.Stats.Filters alias Plausible.Stats.Imported alias Plausible.Stats.Query alias Plausible.Stats.SQL.QueryBuilder - @no_ref "Direct / None" - @not_set "(not set)" - @none "(none)" - @property_to_table_mappings Imported.Base.property_to_table_mappings() @goals_with_url Plausible.Imported.goals_with_url() @@ -302,479 +298,4 @@ defmodule Plausible.Stats.Imported do |> Imported.Base.query_imported(query) |> select_merge([i], %{total_visitors: fragment("sum(?)", i.visitors)}) end - - defp select_imported_metrics(q, []), do: q - - defp select_imported_metrics(q, [:visitors | rest]) do - q - |> select_merge([i], %{visitors: sum(i.visitors)}) - |> select_imported_metrics(rest) - end - - defp select_imported_metrics( - %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_custom_events", _}}} = q, - [:events | rest] - ) do - q - |> select_merge([i], %{events: sum(i.events)}) - |> select_imported_metrics(rest) - end - - defp select_imported_metrics(q, [:events | rest]) do - q - |> select_merge([i], %{events: sum(i.pageviews)}) - |> select_imported_metrics(rest) - end - - defp select_imported_metrics( - %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q, - [:visits | rest] - ) do - q - |> select_merge([i], %{visits: sum(i.exits)}) - |> select_imported_metrics(rest) - end - - defp select_imported_metrics( - %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q, - [:visits | rest] - ) do - q - |> select_merge([i], %{visits: sum(i.entrances)}) - |> select_imported_metrics(rest) - end - - defp select_imported_metrics(q, [:visits | rest]) do - q - |> select_merge([i], %{visits: sum(i.visits)}) - |> select_imported_metrics(rest) - end - - defp select_imported_metrics( - %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_custom_events", _}}} = q, - [:pageviews | rest] - ) do - q - |> select_merge([i], %{pageviews: 0}) - |> select_imported_metrics(rest) - end - - defp select_imported_metrics(q, [:pageviews | rest]) do - q - |> where([i], i.pageviews > 0) - |> select_merge([i], %{pageviews: sum(i.pageviews)}) - |> select_imported_metrics(rest) - end - - defp select_imported_metrics( - %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_pages", _}}} = q, - [:bounce_rate | rest] - ) do - q - |> select_merge([i], %{ - bounces: 0, - __internal_visits: 0 - }) - |> select_imported_metrics(rest) - end - - defp select_imported_metrics( - %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q, - [:bounce_rate | rest] - ) do - q - |> select_merge([i], %{ - bounces: sum(i.bounces), - __internal_visits: sum(i.entrances) - }) - |> select_imported_metrics(rest) - end - - defp select_imported_metrics( - %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q, - [:bounce_rate | rest] - ) do - q - |> select_merge([i], %{ - bounces: sum(i.bounces), - __internal_visits: sum(i.exits) - }) - |> select_imported_metrics(rest) - end - - defp select_imported_metrics(q, [:bounce_rate | rest]) do - q - |> select_merge([i], %{ - bounces: sum(i.bounces), - __internal_visits: sum(i.visits) - }) - |> select_imported_metrics(rest) - end - - defp select_imported_metrics( - %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_pages", _}}} = q, - [:visit_duration | rest] - ) do - q - |> select_merge([i], %{ - visit_duration: 0, - __internal_visits: 0 - }) - |> select_imported_metrics(rest) - end - - defp select_imported_metrics( - %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q, - [:visit_duration | rest] - ) do - q - |> select_merge([i], %{ - visit_duration: sum(i.visit_duration), - __internal_visits: sum(i.entrances) - }) - |> select_imported_metrics(rest) - end - - defp select_imported_metrics( - %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q, - [:visit_duration | rest] - ) do - q - |> select_merge([i], %{ - visit_duration: sum(i.visit_duration), - __internal_visits: sum(i.exits) - }) - |> select_imported_metrics(rest) - end - - defp select_imported_metrics(q, [:visit_duration | rest]) do - q - |> select_merge([i], %{ - visit_duration: sum(i.visit_duration), - __internal_visits: sum(i.visits) - }) - |> select_imported_metrics(rest) - end - - defp select_imported_metrics( - %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q, - [:views_per_visit | rest] - ) do - q - |> where([i], i.pageviews > 0) - |> select_merge([i], %{ - pageviews: sum(i.pageviews), - __internal_visits: sum(i.entrances) - }) - |> select_imported_metrics(rest) - end - - defp select_imported_metrics( - %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q, - [:views_per_visit | rest] - ) do - q - |> where([i], i.pageviews > 0) - |> select_merge([i], %{ - pageviews: sum(i.pageviews), - __internal_visits: sum(i.exits) - }) - |> select_imported_metrics(rest) - end - - defp select_imported_metrics(q, [:views_per_visit | rest]) do - q - |> where([i], i.pageviews > 0) - |> select_merge([i], %{ - pageviews: sum(i.pageviews), - __internal_visits: sum(i.visits) - }) - |> select_imported_metrics(rest) - end - - defp select_imported_metrics(q, [_ | rest]) do - q - |> select_imported_metrics(rest) - end - - defp group_imported_by(q, query) do - Enum.reduce(query.dimensions, q, fn dimension, q -> - dim = Plausible.Stats.Filters.without_prefix(dimension) - - group_imported_by(q, dim, shortname(query, dimension), query) - end) - end - - defp group_imported_by(q, dim, key, _query) when dim in [:source, :referrer] do - q - |> group_by([i], field(i, ^dim)) - |> select_merge_as([i], %{ - key => fragment("if(empty(?), ?, ?)", field(i, ^dim), @no_ref, field(i, ^dim)) - }) - end - - defp group_imported_by(q, dim, key, _query) - when dim in [:utm_source, :utm_medium, :utm_campaign, :utm_term, :utm_content] do - q - |> group_by([i], field(i, ^dim)) - |> where([i], fragment("not empty(?)", field(i, ^dim))) - |> select_merge_as([i], %{key => field(i, ^dim)}) - end - - defp group_imported_by(q, :page, key, _query) do - q - |> group_by([i], i.page) - |> select_merge_as([i], %{key => i.page, time_on_page: sum(i.time_on_page)}) - end - - defp group_imported_by(q, :country, key, _query) do - q - |> group_by([i], i.country) - |> where([i], i.country != "ZZ") - |> select_merge_as([i], %{key => i.country}) - end - - defp group_imported_by(q, :region, key, _query) do - q - |> group_by([i], i.region) - |> where([i], i.region != "") - |> select_merge_as([i], %{key => i.region}) - end - - defp group_imported_by(q, :city, key, _query) do - q - |> group_by([i], i.city) - |> where([i], i.city != 0 and not is_nil(i.city)) - |> select_merge_as([i], %{key => i.city}) - end - - defp group_imported_by(q, dim, key, _query) when dim in [:device, :browser] do - q - |> group_by([i], field(i, ^dim)) - |> select_merge_as([i], %{ - key => fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim)) - }) - end - - defp group_imported_by(q, :browser_version, key, _query) do - q - |> group_by([i], [i.browser_version]) - |> select_merge_as([i], %{ - key => fragment("if(empty(?), ?, ?)", i.browser_version, @not_set, i.browser_version) - }) - end - - defp group_imported_by(q, :os, key, _query) do - q - |> group_by([i], i.operating_system) - |> select_merge_as([i], %{ - key => fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system) - }) - end - - defp group_imported_by(q, :os_version, key, _query) do - q - |> group_by([i], [i.operating_system_version]) - |> select_merge_as([i], %{ - key => - fragment( - "if(empty(?), ?, ?)", - i.operating_system_version, - @not_set, - i.operating_system_version - ) - }) - end - - defp group_imported_by(q, dim, key, _query) when dim in [:entry_page, :exit_page] do - q - |> group_by([i], field(i, ^dim)) - |> select_merge_as([i], %{key => field(i, ^dim)}) - end - - defp group_imported_by(q, :name, key, _query) do - q - |> group_by([i], i.name) - |> select_merge_as([i], %{key => i.name}) - end - - defp group_imported_by(q, :url, key, _query) do - q - |> group_by([i], i.link_url) - |> select_merge_as([i], %{ - key => fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none) - }) - end - - defp group_imported_by(q, :path, key, _query) do - q - |> group_by([i], i.path) - |> select_merge_as([i], %{ - key => fragment("if(not empty(?), ?, ?)", i.path, i.path, @none) - }) - end - - defp group_imported_by(q, :month, key, _query) do - q - |> group_by([i], fragment("toStartOfMonth(?)", i.date)) - |> select_merge_as([i], %{key => fragment("toStartOfMonth(?)", i.date)}) - end - - defp group_imported_by(q, :hour, key, _query) do - q - |> group_by([i], i.date) - |> select_merge_as([i], %{key => i.date}) - end - - defp group_imported_by(q, :week, key, query) do - q - |> group_by([i], weekstart_not_before(i.date, ^query.date_range.first)) - |> select_merge_as([i], %{ - key => weekstart_not_before(i.date, ^query.date_range.first) - }) - end - - defp group_imported_by(q, :day, key, _query) do - q - |> group_by([i], i.date) - |> select_merge_as([i], %{key => i.date}) - end - - defp select_joined_dimensions(q, query) do - Enum.reduce(query.dimensions, q, fn dimension, q -> - select_joined_dimension(q, dimension, shortname(query, dimension)) - end) - end - - defp select_joined_dimension(q, "visit:city", key) do - select_merge_as(q, [s, i], %{ - key => fragment("greatest(?,?)", field(i, ^key), field(s, ^key)) - }) - end - - defp select_joined_dimension(q, "time:" <> _, key) do - select_merge_as(q, [s, i], %{ - key => fragment("greatest(?, ?)", field(i, ^key), field(s, ^key)) - }) - end - - defp select_joined_dimension(q, _dimension, key) do - select_merge_as(q, [s, i], %{ - key => fragment("if(empty(?), ?, ?)", field(s, ^key), field(i, ^key), field(s, ^key)) - }) - end - - defp select_joined_metrics(q, []), do: q - # NOTE: Reverse-engineering the native data bounces and total visit - # durations to combine with imported data is inefficient. Instead both - # queries should fetch bounces/total_visit_duration and visits and be - # used as subqueries to a main query that then find the bounce rate/avg - # visit_duration. - - defp select_joined_metrics(q, [:visits | rest]) do - q - |> select_merge_as([s, i], %{visits: s.visits + i.visits}) - |> select_joined_metrics(rest) - end - - defp select_joined_metrics(q, [:visitors | rest]) do - q - |> select_merge_as([s, i], %{visitors: s.visitors + i.visitors}) - |> select_joined_metrics(rest) - end - - defp select_joined_metrics(q, [:events | rest]) do - q - |> select_merge_as([s, i], %{events: s.events + i.events}) - |> select_joined_metrics(rest) - end - - defp select_joined_metrics(q, [:pageviews | rest]) do - q - |> select_merge_as([s, i], %{pageviews: s.pageviews + i.pageviews}) - |> select_joined_metrics(rest) - end - - defp select_joined_metrics(q, [:views_per_visit | rest]) do - q - |> select_merge_as([s, i], %{ - views_per_visit: - fragment( - "if(? + ? > 0, round((? + ? * ?) / (? + ?), 2), 0)", - s.__internal_visits, - i.__internal_visits, - i.pageviews, - s.views_per_visit, - s.__internal_visits, - i.__internal_visits, - s.__internal_visits - ) - }) - |> select_joined_metrics(rest) - end - - defp select_joined_metrics(q, [:bounce_rate | rest]) do - q - |> select_merge_as([s, i], %{ - bounce_rate: - fragment( - "if(? + ? > 0, round(100 * (? + (? * ? / 100)) / (? + ?)), 0)", - s.__internal_visits, - i.__internal_visits, - i.bounces, - s.bounce_rate, - s.__internal_visits, - i.__internal_visits, - s.__internal_visits - ) - }) - |> select_joined_metrics(rest) - end - - defp select_joined_metrics(q, [:visit_duration | rest]) do - q - |> select_merge_as([s, i], %{ - visit_duration: - fragment( - """ - if( - ? + ? > 0, - round((? + ? * ?) / (? + ?), 0), - 0 - ) - """, - s.__internal_visits, - i.__internal_visits, - i.visit_duration, - s.visit_duration, - s.__internal_visits, - s.__internal_visits, - i.__internal_visits - ) - }) - |> select_joined_metrics(rest) - end - - defp select_joined_metrics(q, [:sample_percent | rest]) do - q - |> select_merge_as([s, i], %{sample_percent: s.sample_percent}) - |> select_joined_metrics(rest) - end - - defp select_joined_metrics(q, [_ | rest]) do - q - |> select_joined_metrics(rest) - end - - defp naive_dimension_join(q1, q2, metrics) do - from(a in subquery(q1), - full_join: b in subquery(q2), - on: a.dim0 == b.dim0, - select: %{} - ) - |> select_merge_as([a, b], %{ - dim0: fragment("if(? != 0, ?, ?)", a.dim0, a.dim0, b.dim0) - }) - |> select_joined_metrics(metrics) - end end diff --git a/lib/plausible/stats/imported/sql/builder.ex b/lib/plausible/stats/imported/sql/builder.ex new file mode 100644 index 000000000..cf81959d3 --- /dev/null +++ b/lib/plausible/stats/imported/sql/builder.ex @@ -0,0 +1,462 @@ +defmodule Plausible.Stats.Imported.SQL.Builder do + @moduledoc """ + This module is responsible for generating SQL/Ecto expressions + for dimensions, filters and metrics used in import table queries + """ + + use Plausible.Stats.SQL.Fragments + + import Plausible.Stats.Util, only: [shortname: 2] + import Ecto.Query + + @no_ref "Direct / None" + @not_set "(not set)" + @none "(none)" + + def select_imported_metrics(q, []), do: q + + def select_imported_metrics(q, [:visitors | rest]) do + q + |> select_merge([i], %{visitors: sum(i.visitors)}) + |> select_imported_metrics(rest) + end + + def select_imported_metrics( + %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_custom_events", _}}} = q, + [:events | rest] + ) do + q + |> select_merge([i], %{events: sum(i.events)}) + |> select_imported_metrics(rest) + end + + def select_imported_metrics(q, [:events | rest]) do + q + |> select_merge([i], %{events: sum(i.pageviews)}) + |> select_imported_metrics(rest) + end + + def select_imported_metrics( + %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q, + [:visits | rest] + ) do + q + |> select_merge([i], %{visits: sum(i.exits)}) + |> select_imported_metrics(rest) + end + + def select_imported_metrics( + %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q, + [:visits | rest] + ) do + q + |> select_merge([i], %{visits: sum(i.entrances)}) + |> select_imported_metrics(rest) + end + + def select_imported_metrics(q, [:visits | rest]) do + q + |> select_merge([i], %{visits: sum(i.visits)}) + |> select_imported_metrics(rest) + end + + def select_imported_metrics( + %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_custom_events", _}}} = q, + [:pageviews | rest] + ) do + q + |> select_merge([i], %{pageviews: 0}) + |> select_imported_metrics(rest) + end + + def select_imported_metrics(q, [:pageviews | rest]) do + q + |> where([i], i.pageviews > 0) + |> select_merge([i], %{pageviews: sum(i.pageviews)}) + |> select_imported_metrics(rest) + end + + def select_imported_metrics( + %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_pages", _}}} = q, + [:bounce_rate | rest] + ) do + q + |> select_merge([i], %{ + bounces: 0, + __internal_visits: 0 + }) + |> select_imported_metrics(rest) + end + + def select_imported_metrics( + %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q, + [:bounce_rate | rest] + ) do + q + |> select_merge([i], %{ + bounces: sum(i.bounces), + __internal_visits: sum(i.entrances) + }) + |> select_imported_metrics(rest) + end + + def select_imported_metrics( + %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q, + [:bounce_rate | rest] + ) do + q + |> select_merge([i], %{ + bounces: sum(i.bounces), + __internal_visits: sum(i.exits) + }) + |> select_imported_metrics(rest) + end + + def select_imported_metrics(q, [:bounce_rate | rest]) do + q + |> select_merge([i], %{ + bounces: sum(i.bounces), + __internal_visits: sum(i.visits) + }) + |> select_imported_metrics(rest) + end + + def select_imported_metrics( + %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_pages", _}}} = q, + [:visit_duration | rest] + ) do + q + |> select_merge([i], %{ + visit_duration: 0, + __internal_visits: 0 + }) + |> select_imported_metrics(rest) + end + + def select_imported_metrics( + %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q, + [:visit_duration | rest] + ) do + q + |> select_merge([i], %{ + visit_duration: sum(i.visit_duration), + __internal_visits: sum(i.entrances) + }) + |> select_imported_metrics(rest) + end + + def select_imported_metrics( + %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q, + [:visit_duration | rest] + ) do + q + |> select_merge([i], %{ + visit_duration: sum(i.visit_duration), + __internal_visits: sum(i.exits) + }) + |> select_imported_metrics(rest) + end + + def select_imported_metrics(q, [:visit_duration | rest]) do + q + |> select_merge([i], %{ + visit_duration: sum(i.visit_duration), + __internal_visits: sum(i.visits) + }) + |> select_imported_metrics(rest) + end + + def select_imported_metrics( + %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q, + [:views_per_visit | rest] + ) do + q + |> where([i], i.pageviews > 0) + |> select_merge([i], %{ + pageviews: sum(i.pageviews), + __internal_visits: sum(i.entrances) + }) + |> select_imported_metrics(rest) + end + + def select_imported_metrics( + %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q, + [:views_per_visit | rest] + ) do + q + |> where([i], i.pageviews > 0) + |> select_merge([i], %{ + pageviews: sum(i.pageviews), + __internal_visits: sum(i.exits) + }) + |> select_imported_metrics(rest) + end + + def select_imported_metrics(q, [:views_per_visit | rest]) do + q + |> where([i], i.pageviews > 0) + |> select_merge([i], %{ + pageviews: sum(i.pageviews), + __internal_visits: sum(i.visits) + }) + |> select_imported_metrics(rest) + end + + def select_imported_metrics(q, [_ | rest]) do + q + |> select_imported_metrics(rest) + end + + def group_imported_by(q, query) do + Enum.reduce(query.dimensions, q, fn dimension, q -> + q + |> select_group_fields(dimension, shortname(query, dimension), query) + |> filter_group_values(dimension) + |> group_by([], selected_as(^shortname(query, dimension))) + end) + end + + defp select_group_fields(q, dimension, key, _query) + when dimension in ["visit:source", "visit:referrer"] do + select_merge_as(q, [i], %{ + key => + fragment( + "if(empty(?), ?, ?)", + field(i, ^dim(dimension)), + @no_ref, + field(i, ^dim(dimension)) + ) + }) + end + + defp select_group_fields(q, "event:page", key, _query) do + select_merge_as(q, [i], %{key => i.page, time_on_page: sum(i.time_on_page)}) + end + + defp select_group_fields(q, dimension, key, _query) + when dimension in ["visit:device", "visit:browser"] do + select_merge_as(q, [i], %{ + key => + fragment( + "if(empty(?), ?, ?)", + field(i, ^dim(dimension)), + @not_set, + field(i, ^dim(dimension)) + ) + }) + end + + defp select_group_fields(q, "visit:browser_version", key, _query) do + select_merge_as(q, [i], %{ + key => fragment("if(empty(?), ?, ?)", i.browser_version, @not_set, i.browser_version) + }) + end + + defp select_group_fields(q, "visit:os", key, _query) do + select_merge_as(q, [i], %{ + key => fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system) + }) + end + + defp select_group_fields(q, "visit:os_version", key, _query) do + select_merge_as(q, [i], %{ + key => + fragment( + "if(empty(?), ?, ?)", + i.operating_system_version, + @not_set, + i.operating_system_version + ) + }) + end + + defp select_group_fields(q, "event:props:url", key, _query) do + select_merge_as(q, [i], %{ + key => fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none) + }) + end + + defp select_group_fields(q, "event:props:path", key, _query) do + select_merge_as(q, [i], %{ + key => fragment("if(not empty(?), ?, ?)", i.path, i.path, @none) + }) + end + + defp select_group_fields(q, "time:month", key, _query) do + select_merge_as(q, [i], %{key => fragment("toStartOfMonth(?)", i.date)}) + end + + defp select_group_fields(q, dimension, key, _query) + when dimension in ["time:hour", "time:day"] do + select_merge_as(q, [i], %{key => i.date}) + end + + defp select_group_fields(q, "time:week", key, query) do + select_merge_as(q, [i], %{ + key => weekstart_not_before(i.date, ^query.date_range.first) + }) + end + + defp select_group_fields(q, dimension, key, _query) do + select_merge_as(q, [i], %{key => field(i, ^dim(dimension))}) + end + + @utm_dimensions [ + "visit:utm_source", + "visit:utm_medium", + "visit:utm_campaign", + "visit:utm_term", + "visit:utm_content" + ] + defp filter_group_values(q, dimension) when dimension in @utm_dimensions do + dim = Plausible.Stats.Filters.without_prefix(dimension) + + where(q, [i], fragment("not empty(?)", field(i, ^dim))) + end + + defp filter_group_values(q, "visit:country"), do: where(q, [i], i.country != "ZZ") + defp filter_group_values(q, "visit:region"), do: where(q, [i], i.region != "") + defp filter_group_values(q, "visit:city"), do: where(q, [i], i.city != 0 and not is_nil(i.city)) + + defp filter_group_values(q, _dimension), do: q + + def select_joined_dimensions(q, query) do + Enum.reduce(query.dimensions, q, fn dimension, q -> + select_joined_dimension(q, dimension, shortname(query, dimension)) + end) + end + + defp select_joined_dimension(q, "visit:city", key) do + select_merge_as(q, [s, i], %{ + key => fragment("greatest(?,?)", field(i, ^key), field(s, ^key)) + }) + end + + defp select_joined_dimension(q, "time:" <> _, key) do + select_merge_as(q, [s, i], %{ + key => fragment("greatest(?, ?)", field(i, ^key), field(s, ^key)) + }) + end + + defp select_joined_dimension(q, _dimension, key) do + select_merge_as(q, [s, i], %{ + key => fragment("if(empty(?), ?, ?)", field(s, ^key), field(i, ^key), field(s, ^key)) + }) + end + + def select_joined_metrics(q, []), do: q + # NOTE: Reverse-engineering the native data bounces and total visit + # durations to combine with imported data is inefficient. Instead both + # queries should fetch bounces/total_visit_duration and visits and be + # used as subqueries to a main query that then find the bounce rate/avg + # visit_duration. + + def select_joined_metrics(q, [:visits | rest]) do + q + |> select_merge_as([s, i], %{visits: s.visits + i.visits}) + |> select_joined_metrics(rest) + end + + def select_joined_metrics(q, [:visitors | rest]) do + q + |> select_merge_as([s, i], %{visitors: s.visitors + i.visitors}) + |> select_joined_metrics(rest) + end + + def select_joined_metrics(q, [:events | rest]) do + q + |> select_merge_as([s, i], %{events: s.events + i.events}) + |> select_joined_metrics(rest) + end + + def select_joined_metrics(q, [:pageviews | rest]) do + q + |> select_merge_as([s, i], %{pageviews: s.pageviews + i.pageviews}) + |> select_joined_metrics(rest) + end + + def select_joined_metrics(q, [:views_per_visit | rest]) do + q + |> select_merge_as([s, i], %{ + views_per_visit: + fragment( + "if(? + ? > 0, round((? + ? * ?) / (? + ?), 2), 0)", + s.__internal_visits, + i.__internal_visits, + i.pageviews, + s.views_per_visit, + s.__internal_visits, + i.__internal_visits, + s.__internal_visits + ) + }) + |> select_joined_metrics(rest) + end + + def select_joined_metrics(q, [:bounce_rate | rest]) do + q + |> select_merge_as([s, i], %{ + bounce_rate: + fragment( + "if(? + ? > 0, round(100 * (? + (? * ? / 100)) / (? + ?)), 0)", + s.__internal_visits, + i.__internal_visits, + i.bounces, + s.bounce_rate, + s.__internal_visits, + i.__internal_visits, + s.__internal_visits + ) + }) + |> select_joined_metrics(rest) + end + + def select_joined_metrics(q, [:visit_duration | rest]) do + q + |> select_merge_as([s, i], %{ + visit_duration: + fragment( + """ + if( + ? + ? > 0, + round((? + ? * ?) / (? + ?), 0), + 0 + ) + """, + s.__internal_visits, + i.__internal_visits, + i.visit_duration, + s.visit_duration, + s.__internal_visits, + s.__internal_visits, + i.__internal_visits + ) + }) + |> select_joined_metrics(rest) + end + + def select_joined_metrics(q, [:sample_percent | rest]) do + q + |> select_merge_as([s, i], %{sample_percent: s.sample_percent}) + |> select_joined_metrics(rest) + end + + def select_joined_metrics(q, [_ | rest]) do + q + |> select_joined_metrics(rest) + end + + def naive_dimension_join(q1, q2, metrics) do + from(a in subquery(q1), + full_join: b in subquery(q2), + on: a.dim0 == b.dim0, + select: %{} + ) + |> select_merge_as([a, b], %{ + dim0: fragment("if(? != 0, ?, ?)", a.dim0, a.dim0, b.dim0) + }) + |> select_joined_metrics(metrics) + end + + defp dim(dimension), do: Plausible.Stats.Filters.without_prefix(dimension) +end diff --git a/lib/plausible/stats/props.ex b/lib/plausible/stats/legacy/legacy_dimensions.ex similarity index 58% rename from lib/plausible/stats/props.ex rename to lib/plausible/stats/legacy/legacy_dimensions.ex index c0f1bff9d..3de9c023a 100644 --- a/lib/plausible/stats/props.ex +++ b/lib/plausible/stats/legacy/legacy_dimensions.ex @@ -1,4 +1,7 @@ -defmodule Plausible.Stats.Props do +defmodule Plausible.Stats.Legacy.Dimensions do + @moduledoc """ + Deprecated module. See QueryParser for list of valid dimensions + """ @event_props ["event:page", "event:name", "event:goal", "event:hostname"] @session_props [ "visit:source", @@ -20,8 +23,8 @@ defmodule Plausible.Stats.Props do "visit:browser_version" ] - def valid_prop?(prop) when prop in @event_props, do: true - def valid_prop?(prop) when prop in @session_props, do: true - def valid_prop?("event:props:" <> prop) when byte_size(prop) > 0, do: true - def valid_prop?(_), do: false + def valid?(prop) when prop in @event_props, do: true + def valid?(prop) when prop in @session_props, do: true + def valid?("event:props:" <> prop) when byte_size(prop) > 0, do: true + def valid?(_), do: false end diff --git a/lib/plausible/stats/legacy/legacy_query_builder.ex b/lib/plausible/stats/legacy/legacy_query_builder.ex new file mode 100644 index 000000000..9ec14a713 --- /dev/null +++ b/lib/plausible/stats/legacy/legacy_query_builder.ex @@ -0,0 +1,180 @@ +defmodule Plausible.Stats.Legacy.QueryBuilder do + @moduledoc false + + use Plausible + + alias Plausible.Stats.{Filters, Interval, Query} + + def from(site, params) do + now = NaiveDateTime.utc_now(:second) + + query = + Query + |> struct!(now: now, timezone: site.timezone) + |> put_period(site, params) + |> put_dimensions(params) + |> put_interval(params) + |> put_parsed_filters(params) + |> Query.put_experimental_reduced_joins(site, params) + |> Query.put_imported_opts(site, params) + + on_ee do + query = Plausible.Stats.Sampling.put_threshold(query, params) + end + + query + end + + defp put_period(query, site, %{"period" => "realtime"}) do + date = today(site.timezone) + + struct!(query, period: "realtime", date_range: Date.range(date, date)) + end + + defp put_period(query, site, %{"period" => "day"} = params) do + date = parse_single_date(site.timezone, params) + + struct!(query, period: "day", date_range: Date.range(date, date)) + end + + defp put_period(query, site, %{"period" => "7d"} = params) do + end_date = parse_single_date(site.timezone, params) + start_date = end_date |> Timex.shift(days: -6) + + struct!( + query, + period: "7d", + date_range: Date.range(start_date, end_date) + ) + end + + defp put_period(query, site, %{"period" => "30d"} = params) do + end_date = parse_single_date(site.timezone, params) + start_date = end_date |> Timex.shift(days: -30) + + struct!(query, period: "30d", date_range: Date.range(start_date, end_date)) + end + + defp put_period(query, site, %{"period" => "month"} = params) do + date = parse_single_date(site.timezone, params) + + start_date = Timex.beginning_of_month(date) + end_date = Timex.end_of_month(date) + + struct!(query, + period: "month", + date_range: Date.range(start_date, end_date) + ) + end + + defp put_period(query, site, %{"period" => "6mo"} = params) do + end_date = + parse_single_date(site.timezone, params) + |> Timex.end_of_month() + + start_date = + Timex.shift(end_date, months: -5) + |> Timex.beginning_of_month() + + struct!(query, + period: "6mo", + date_range: Date.range(start_date, end_date) + ) + end + + defp put_period(query, site, %{"period" => "12mo"} = params) do + end_date = + parse_single_date(site.timezone, params) + |> Timex.end_of_month() + + start_date = + Timex.shift(end_date, months: -11) + |> Timex.beginning_of_month() + + struct!(query, + period: "12mo", + date_range: Date.range(start_date, end_date) + ) + end + + defp put_period(query, site, %{"period" => "year"} = params) do + end_date = + parse_single_date(site.timezone, params) + |> Timex.end_of_year() + + start_date = Timex.beginning_of_year(end_date) + + struct!(query, + period: "year", + date_range: Date.range(start_date, end_date) + ) + end + + defp put_period(query, site, %{"period" => "all"}) do + now = today(site.timezone) + start_date = Plausible.Sites.stats_start_date(site) || now + + struct!(query, + period: "all", + date_range: Date.range(start_date, now) + ) + end + + defp put_period(query, site, %{"period" => "custom", "from" => from, "to" => to} = params) do + new_params = + params + |> Map.drop(["from", "to"]) + |> Map.put("date", Enum.join([from, to], ",")) + + put_period(query, site, new_params) + end + + defp put_period(query, _site, %{"period" => "custom", "date" => date}) do + [from, to] = String.split(date, ",") + from_date = Date.from_iso8601!(String.trim(from)) + to_date = Date.from_iso8601!(String.trim(to)) + + struct!(query, + period: "custom", + date_range: Date.range(from_date, to_date) + ) + end + + defp put_period(query, site, params) do + put_period(query, site, Map.merge(params, %{"period" => "30d"})) + end + + defp put_dimensions(query, params) do + if not is_nil(params["property"]) do + struct!(query, dimensions: [params["property"]]) + else + struct!(query, dimensions: Map.get(params, "dimensions", [])) + end + end + + defp put_interval(%{:period => "all"} = query, params) do + interval = Map.get(params, "interval", Interval.default_for_date_range(query.date_range)) + struct!(query, interval: interval) + end + + defp put_interval(query, params) do + interval = Map.get(params, "interval", Interval.default_for_period(query.period)) + struct!(query, interval: interval) + end + + defp put_parsed_filters(query, params) do + struct!(query, filters: Filters.parse(params["filters"])) + end + + defp today(tz) do + Timex.now(tz) |> Timex.to_date() + end + + defp parse_single_date(tz, params) do + case params["date"] do + "today" -> Timex.now(tz) |> Timex.to_date() + date when is_binary(date) -> Date.from_iso8601!(date) + _ -> today(tz) + end + end +end diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index 9d2eb0d66..7de4e931b 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -24,44 +24,31 @@ defmodule Plausible.Stats.Query do } require OpenTelemetry.Tracer, as: Tracer - alias Plausible.Stats.{Filters, Interval, Imported} + alias Plausible.Stats.{Filters, Imported, Legacy} @type t :: %__MODULE__{} - @spec from(Plausible.Site.t(), map()) :: t() - def from(site, params) do - now = NaiveDateTime.utc_now(:second) - - query = - __MODULE__ - |> struct!(now: now, timezone: site.timezone) - |> put_experimental_reduced_joins(site, params) - |> put_period(site, params) - |> put_dimensions(params) - |> put_interval(params) - |> put_parsed_filters(params) - |> put_imported_opts(site, params) - - on_ee do - query = Plausible.Stats.Sampling.put_threshold(query, params) - end - - query - end - def build(site, params) do with {:ok, query_data} <- Filters.QueryParser.parse(site, params) do query = struct!(__MODULE__, Map.to_list(query_data)) |> put_imported_opts(site, %{}) |> put_experimental_reduced_joins(site, params) - |> struct!(v2: true) + |> struct!(v2: true, now: NaiveDateTime.utc_now(:second)) {:ok, query} end end - defp put_experimental_reduced_joins(query, site, params) do + @doc """ + Builds query from old-style params. New code should prefer Query.build + """ + @spec from(Plausible.Site.t(), map()) :: t() + def from(site, params) do + Legacy.QueryBuilder.from(site, params) + end + + def put_experimental_reduced_joins(query, site, params) do if Map.has_key?(params, "experimental_reduced_joins") do struct!(query, experimental_reduced_joins?: Map.get(params, "experimental_reduced_joins") == "true" @@ -73,147 +60,6 @@ defmodule Plausible.Stats.Query do end end - defp put_period(query, site, %{"period" => "realtime"}) do - date = today(site.timezone) - - struct!(query, period: "realtime", date_range: Date.range(date, date)) - end - - defp put_period(query, site, %{"period" => "day"} = params) do - date = parse_single_date(site.timezone, params) - - struct!(query, period: "day", date_range: Date.range(date, date)) - end - - defp put_period(query, site, %{"period" => "7d"} = params) do - end_date = parse_single_date(site.timezone, params) - start_date = end_date |> Timex.shift(days: -6) - - struct!( - query, - period: "7d", - date_range: Date.range(start_date, end_date) - ) - end - - defp put_period(query, site, %{"period" => "30d"} = params) do - end_date = parse_single_date(site.timezone, params) - start_date = end_date |> Timex.shift(days: -30) - - struct!(query, period: "30d", date_range: Date.range(start_date, end_date)) - end - - defp put_period(query, site, %{"period" => "month"} = params) do - date = parse_single_date(site.timezone, params) - - start_date = Timex.beginning_of_month(date) - end_date = Timex.end_of_month(date) - - struct!(query, - period: "month", - date_range: Date.range(start_date, end_date) - ) - end - - defp put_period(query, site, %{"period" => "6mo"} = params) do - end_date = - parse_single_date(site.timezone, params) - |> Timex.end_of_month() - - start_date = - Timex.shift(end_date, months: -5) - |> Timex.beginning_of_month() - - struct!(query, - period: "6mo", - date_range: Date.range(start_date, end_date) - ) - end - - defp put_period(query, site, %{"period" => "12mo"} = params) do - end_date = - parse_single_date(site.timezone, params) - |> Timex.end_of_month() - - start_date = - Timex.shift(end_date, months: -11) - |> Timex.beginning_of_month() - - struct!(query, - period: "12mo", - date_range: Date.range(start_date, end_date) - ) - end - - defp put_period(query, site, %{"period" => "year"} = params) do - end_date = - parse_single_date(site.timezone, params) - |> Timex.end_of_year() - - start_date = Timex.beginning_of_year(end_date) - - struct!(query, - period: "year", - date_range: Date.range(start_date, end_date) - ) - end - - defp put_period(query, site, %{"period" => "all"}) do - now = today(site.timezone) - start_date = Plausible.Sites.stats_start_date(site) || now - - struct!(query, - period: "all", - date_range: Date.range(start_date, now) - ) - end - - defp put_period(query, site, %{"period" => "custom", "from" => from, "to" => to} = params) do - new_params = - params - |> Map.drop(["from", "to"]) - |> Map.put("date", Enum.join([from, to], ",")) - - put_period(query, site, new_params) - end - - defp put_period(query, _site, %{"period" => "custom", "date" => date}) do - [from, to] = String.split(date, ",") - from_date = Date.from_iso8601!(String.trim(from)) - to_date = Date.from_iso8601!(String.trim(to)) - - struct!(query, - period: "custom", - date_range: Date.range(from_date, to_date) - ) - end - - defp put_period(query, site, params) do - put_period(query, site, Map.merge(params, %{"period" => "30d"})) - end - - defp put_dimensions(query, params) do - if not is_nil(params["property"]) do - struct!(query, dimensions: [params["property"]]) - else - struct!(query, dimensions: Map.get(params, "dimensions", [])) - end - end - - defp put_interval(%{:period => "all"} = query, params) do - interval = Map.get(params, "interval", Interval.default_for_date_range(query.date_range)) - struct!(query, interval: interval) - end - - defp put_interval(query, params) do - interval = Map.get(params, "interval", Interval.default_for_period(query.period)) - struct!(query, interval: interval) - end - - defp put_parsed_filters(query, params) do - struct!(query, filters: Filters.parse(params["filters"])) - end - def set(query, keywords) do new_query = struct!(query, keywords) @@ -224,26 +70,7 @@ defmodule Plausible.Stats.Query do end end - @spec set_dimensions(t(), list(String.t())) :: t() - def set_dimensions(query, dimensions) do - query - |> struct!(dimensions: dimensions) - |> refresh_imported_opts() - end - - def set_metrics(query, metrics) do - query - |> struct!(metrics: metrics) - |> refresh_imported_opts() - end - - def set_order_by(query, order_by) do - query - |> struct!(order_by: order_by) - |> refresh_imported_opts() - end - - def put_filter(query, filter) do + def add_filter(query, filter) do query |> struct!(filters: query.filters ++ [filter]) |> refresh_imported_opts() @@ -260,13 +87,6 @@ defmodule Plausible.Stats.Query do |> refresh_imported_opts() end - def exclude_imported(query) do - struct!(query, - include_imported: false, - skip_imported_reason: :manual_exclusion - ) - end - defp refresh_imported_opts(query) do put_imported_opts(query, nil, %{}) end @@ -289,19 +109,7 @@ defmodule Plausible.Stats.Query do end) end - defp today(tz) do - Timex.now(tz) |> Timex.to_date() - end - - defp parse_single_date(tz, params) do - case params["date"] do - "today" -> Timex.now(tz) |> Timex.to_date() - date when is_binary(date) -> Date.from_iso8601!(date) - _ -> today(tz) - end - end - - defp put_imported_opts(query, site, params) do + def put_imported_opts(query, site, params) do requested? = params["with_imported"] == "true" || query.include.imports latest_import_end_date = diff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.ex index ef61a5975..aaf3eb8c9 100644 --- a/lib/plausible/stats/sql/query_builder.ex +++ b/lib/plausible/stats/sql/query_builder.ex @@ -32,7 +32,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do from( e in "events_v2", where: ^SQL.WhereBuilder.build(:events, site, events_query), - select: ^Base.select_event_metrics(events_query.metrics) + select: ^select_event_metrics(events_query) ) on_ee do @@ -73,7 +73,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do from( e in "sessions_v2", where: ^SQL.WhereBuilder.build(:sessions, site, sessions_query), - select: ^Base.select_session_metrics(sessions_query.metrics, sessions_query) + select: ^select_session_metrics(sessions_query) ) on_ee do @@ -111,6 +111,18 @@ defmodule Plausible.Stats.SQL.QueryBuilder do end end + defp select_event_metrics(query) do + query.metrics + |> Enum.map(&SQL.Expression.event_metric/1) + |> Enum.reduce(%{}, &Map.merge/2) + end + + defp select_session_metrics(query) do + query.metrics + |> Enum.map(&SQL.Expression.session_metric(&1, query)) + |> Enum.reduce(%{}, &Map.merge/2) + end + defp build_group_by(q, table, query) do Enum.reduce(query.dimensions, q, &dimension_group_by(&2, table, query, &1)) end diff --git a/lib/plausible/stats/sql/where_builder.ex b/lib/plausible/stats/sql/where_builder.ex index 73eeecaee..5f882e060 100644 --- a/lib/plausible/stats/sql/where_builder.ex +++ b/lib/plausible/stats/sql/where_builder.ex @@ -4,7 +4,8 @@ defmodule Plausible.Stats.SQL.WhereBuilder do """ import Ecto.Query - import Plausible.Stats.Base, only: [page_regex: 1, utc_boundaries: 2] + import Plausible.Stats.Time, only: [utc_boundaries: 2] + import Plausible.Stats.Filters.Utils, only: [page_regex: 1] alias Plausible.Stats.Query diff --git a/lib/plausible/stats/time.ex b/lib/plausible/stats/time.ex index c8c0d7a71..83296bffa 100644 --- a/lib/plausible/stats/time.ex +++ b/lib/plausible/stats/time.ex @@ -4,6 +4,57 @@ defmodule Plausible.Stats.Time do """ alias Plausible.Stats.Query + alias Plausible.Timezones + + def utc_boundaries(%Query{period: "realtime", now: now}, site) do + last_datetime = + now + |> Timex.shift(seconds: 5) + |> beginning_of_time(site.native_stats_start_at) + |> NaiveDateTime.truncate(:second) + + first_datetime = + now |> Timex.shift(minutes: -5) |> NaiveDateTime.truncate(:second) + + {first_datetime, last_datetime} + end + + def utc_boundaries(%Query{period: "30m", now: now}, site) do + last_datetime = + now + |> Timex.shift(seconds: 5) + |> beginning_of_time(site.native_stats_start_at) + |> NaiveDateTime.truncate(:second) + + first_datetime = + now |> Timex.shift(minutes: -30) |> NaiveDateTime.truncate(:second) + + {first_datetime, last_datetime} + end + + def utc_boundaries(%Query{date_range: date_range}, site) do + {:ok, first} = NaiveDateTime.new(date_range.first, ~T[00:00:00]) + + first_datetime = + first + |> Timezones.to_utc_datetime(site.timezone) + |> beginning_of_time(site.native_stats_start_at) + + {:ok, last} = NaiveDateTime.new(date_range.last |> Timex.shift(days: 1), ~T[00:00:00]) + + last_datetime = Timezones.to_utc_datetime(last, site.timezone) + + {first_datetime, last_datetime} + end + + defp beginning_of_time(candidate, native_stats_start_at) do + if Timex.after?(native_stats_start_at, candidate) do + native_stats_start_at + else + candidate + end + end + def format_datetime(%Date{} = date), do: Date.to_string(date) def format_datetime(%DateTime{} = datetime), diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex index 5fc3938ed..513debc59 100644 --- a/lib/plausible/stats/timeseries.ex +++ b/lib/plausible/stats/timeseries.ex @@ -1,4 +1,10 @@ defmodule Plausible.Stats.Timeseries do + @moduledoc """ + Builds timeseries results for v1 of our stats API and dashboards. + + Avoid adding new logic here - update QueryBuilder etc instead. + """ + use Plausible use Plausible.ClickhouseRepo alias Plausible.Stats.{Query, QueryOptimizer, QueryResult, SQL} diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex index 248f709ba..e282ca97b 100644 --- a/lib/plausible_web/controllers/api/external_stats_controller.ex +++ b/lib/plausible_web/controllers/api/external_stats_controller.ex @@ -76,7 +76,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do {:error, "Property 'event:hostname' is currently not supported for breakdowns. Please provide a valid property for the breakdown endpoint: https://plausible.io/docs/stats-api#properties"} - Plausible.Stats.Props.valid_prop?(property) -> + Plausible.Stats.Legacy.Dimensions.valid?(property) -> :ok true -> @@ -363,7 +363,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do end defp validate_filter(_site, [_, property | _]) do - if Plausible.Stats.Props.valid_prop?(property) do + if Plausible.Stats.Legacy.Dimensions.valid?(property) do :ok else {:error, diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index e5f01eda3..b3beef0e2 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -777,7 +777,7 @@ defmodule PlausibleWeb.Api.StatsController do query = Query.from(site, params) - |> Query.put_filter(referrer_filter) + |> Query.add_filter(referrer_filter) pagination = parse_pagination(params) @@ -907,9 +907,9 @@ defmodule PlausibleWeb.Api.StatsController do total_pageviews_query = query |> Query.remove_filters(["visit:exit_page"]) - |> Query.put_filter([:is, "event:page", pages]) - |> Query.put_filter([:is, "event:name", ["pageview"]]) - |> Query.set_dimensions(["event:page"]) + |> Query.add_filter([:is, "event:page", pages]) + |> Query.add_filter([:is, "event:name", ["pageview"]]) + |> Query.set(dimensions: ["event:page"]) total_pageviews = Stats.breakdown(site, total_pageviews_query, [:pageviews], {limit, 1}) diff --git a/test/plausible/stats/legacy_dashboard_filter_parser_test.exs b/test/plausible/stats/legacy_dashboard_filter_parser_test.exs index 8a3e94053..ffdcc558a 100644 --- a/test/plausible/stats/legacy_dashboard_filter_parser_test.exs +++ b/test/plausible/stats/legacy_dashboard_filter_parser_test.exs @@ -1,4 +1,4 @@ -defmodule Plausible.Stats.LegacyDashboardFilterParserTest do +defmodule Plausible.Stats.Legacy.DashboardFilterParserTest do use ExUnit.Case, async: true alias Plausible.Stats.Filters.LegacyDashboardFilterParser diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exs index 62c919319..cd6a60668 100644 --- a/test/plausible/stats/query_test.exs +++ b/test/plausible/stats/query_test.exs @@ -21,11 +21,11 @@ defmodule Plausible.Stats.QueryTest do } do q1 = %{now: %NaiveDateTime{}} = Query.from(site, %{"period" => "realtime"}) q2 = %{now: %NaiveDateTime{}} = Query.from(site, %{"period" => "30m"}) - boundaries1 = Plausible.Stats.Base.utc_boundaries(q1, site) - boundaries2 = Plausible.Stats.Base.utc_boundaries(q2, site) + boundaries1 = Plausible.Stats.Time.utc_boundaries(q1, site) + boundaries2 = Plausible.Stats.Time.utc_boundaries(q2, site) :timer.sleep(1500) - assert ^boundaries1 = Plausible.Stats.Base.utc_boundaries(q1, site) - assert ^boundaries2 = Plausible.Stats.Base.utc_boundaries(q2, site) + assert ^boundaries1 = Plausible.Stats.Time.utc_boundaries(q1, site) + assert ^boundaries2 = Plausible.Stats.Time.utc_boundaries(q2, site) end test "parses day format", %{site: site} do diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_goal_dimension_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_goal_dimension_test.exs new file mode 100644 index 000000000..643394e4f --- /dev/null +++ b/test/plausible_web/controllers/api/external_stats_controller/query_goal_dimension_test.exs @@ -0,0 +1,133 @@ +defmodule PlausibleWeb.Api.ExternalStatsController.QueryGoalDimensionTest do + use PlausibleWeb.ConnCase + + @user_id 1231 + + setup [:create_user, :create_new_site, :create_api_key, :use_api_key] + + describe "breakdown by event:goal" do + test "returns custom event goals and pageview goals", %{conn: conn, site: site} do + insert(:goal, %{site: site, event_name: "Purchase"}) + insert(:goal, %{site: site, page_path: "/test"}) + + populate_stats(site, [ + build(:pageview, + timestamp: ~N[2021-01-01 00:00:01], + pathname: "/test" + ), + build(:event, + name: "Purchase", + timestamp: ~N[2021-01-01 00:00:03] + ), + build(:event, + name: "Purchase", + timestamp: ~N[2021-01-01 00:00:03] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors"], + "dimensions" => ["event:goal"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Purchase"], "metrics" => [2]}, + %{"dimensions" => ["Visit /test"], "metrics" => [1]} + ] + end + + test "returns pageview goals containing wildcards", %{conn: conn, site: site} do + insert(:goal, %{site: site, page_path: "/**/post"}) + insert(:goal, %{site: site, page_path: "/blog**"}) + + populate_stats(site, [ + build(:pageview, pathname: "/blog", user_id: @user_id), + build(:pageview, pathname: "/blog/post-1", user_id: @user_id), + build(:pageview, pathname: "/blog/post-2", user_id: @user_id), + build(:pageview, pathname: "/blog/something/post"), + build(:pageview, pathname: "/different/page/post") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "pageviews"], + "dimensions" => ["event:goal"], + "order_by" => [["pageviews", "desc"]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Visit /blog**"], "metrics" => [2, 4]}, + %{"dimensions" => ["Visit /**/post"], "metrics" => [2, 2]} + ] + end + + test "does not return goals that are not configured for the site", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/register"), + build(:event, name: "Signup") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "pageviews"], + "dimensions" => ["event:goal"] + }) + + assert json_response(conn, 200)["results"] == [] + end + + test "returns conversion_rate in an event:goal breakdown", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, name: "Signup", user_id: 1), + build(:event, name: "Signup", user_id: 1), + build(:pageview, pathname: "/blog"), + build(:pageview, pathname: "/blog/post"), + build(:pageview) + ]) + + insert(:goal, %{site: site, event_name: "Signup"}) + insert(:goal, %{site: site, page_path: "/blog**"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "events", "conversion_rate"], + "dimensions" => ["event:goal"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Visit /blog**"], "metrics" => [2, 2, 50.0]}, + %{"dimensions" => ["Signup"], "metrics" => [1, 2, 25.0]} + ] + end + + test "returns conversion_rate alone in an event:goal breakdown", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, name: "Signup", user_id: 1), + build(:pageview) + ]) + + insert(:goal, %{site: site, event_name: "Signup"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["conversion_rate"], + "dimensions" => ["event:goal"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Signup"], "metrics" => [50.0]} + ] + end + end +end diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs new file mode 100644 index 000000000..2e395a6ac --- /dev/null +++ b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs @@ -0,0 +1,695 @@ +defmodule PlausibleWeb.Api.ExternalStatsController.QueryImportedTest do + use PlausibleWeb.ConnCase + + @user_id 1231 + + setup [:create_user, :create_new_site, :create_api_key, :use_api_key] + + describe "aggregation with imported data" do + setup :create_site_import + + test "does not count imported stats unless specified", %{ + conn: conn, + site: site, + site_import: site_import + } do + populate_stats(site, site_import.id, [ + build(:imported_visitors, date: ~D[2023-01-01]), + build(:pageview, timestamp: ~N[2023-01-01 00:00:00]) + ]) + + query_params = %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews"] + } + + conn1 = post(conn, "/api/v2/query", query_params) + + assert json_response(conn1, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}] + + conn2 = post(conn, "/api/v2/query", Map.put(query_params, "include", %{"imports" => true})) + + assert json_response(conn2, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}] + refute json_response(conn2, 200)["meta"]["warning"] + end + end + + test "breaks down all metrics by visit:referrer with imported data", %{conn: conn, site: site} do + site_import = + insert(:site_import, + site: site, + start_date: ~D[2005-01-01], + end_date: Timex.today(), + source: :universal_analytics + ) + + populate_stats(site, site_import.id, [ + build(:pageview, referrer: "site.com", timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, referrer: "site.com/1", timestamp: ~N[2021-01-01 00:00:00]), + build(:imported_sources, + referrer: "site.com", + date: ~D[2021-01-01], + visitors: 2, + visits: 2, + pageviews: 2, + bounces: 1, + visit_duration: 120 + ), + build(:imported_sources, + referrer: "site.com/2", + date: ~D[2021-01-01], + visitors: 2, + visits: 2, + pageviews: 2, + bounces: 2, + visit_duration: 0 + ), + build(:imported_sources, + date: ~D[2021-01-01], + visitors: 10, + visits: 11, + pageviews: 50, + bounces: 0, + visit_duration: 1100 + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "visits", "pageviews", "bounce_rate", "visit_duration"], + "date_range" => "all", + "dimensions" => ["visit:referrer"], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Direct / None"], "metrics" => [10, 11, 50, 0.0, 100.0]}, + %{"dimensions" => ["site.com"], "metrics" => [3, 3, 3, 67.0, 40.0]}, + %{"dimensions" => ["site.com/2"], "metrics" => [2, 2, 2, 100.0, 0.0]}, + %{"dimensions" => ["site.com/1"], "metrics" => [1, 1, 1, 100.0, 0.0]} + ] + end + + test "breaks down all metrics by visit:utm_source with imported data", %{conn: conn, site: site} do + site_import = + insert(:site_import, + site: site, + start_date: ~D[2005-01-01], + end_date: Timex.today(), + source: :universal_analytics + ) + + populate_stats(site, site_import.id, [ + build(:pageview, utm_source: "SomeUTMSource", timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, utm_source: "SomeUTMSource-1", timestamp: ~N[2021-01-01 00:00:00]), + build(:imported_sources, + utm_source: "SomeUTMSource", + date: ~D[2021-01-01], + visitors: 2, + visits: 2, + pageviews: 2, + bounces: 1, + visit_duration: 120 + ), + build(:imported_sources, + utm_source: "SomeUTMSource-2", + date: ~D[2021-01-01], + visitors: 2, + visits: 2, + pageviews: 2, + bounces: 2, + visit_duration: 0 + ), + build(:imported_sources, + date: ~D[2021-01-01], + visitors: 10, + visits: 11, + pageviews: 50, + bounces: 0, + visit_duration: 1100 + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "visits", "pageviews", "bounce_rate", "visit_duration"], + "date_range" => "all", + "dimensions" => ["visit:utm_source"], + "include" => %{"imports" => true} + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["SomeUTMSource"], "metrics" => [3, 3, 3, 67.0, 40.0]}, + %{"dimensions" => ["SomeUTMSource-2"], "metrics" => [2, 2, 2, 100.0, 0.0]}, + %{"dimensions" => ["SomeUTMSource-1"], "metrics" => [1, 1, 1, 100.0, 0.0]} + ] + end + + test "pageviews breakdown by event:page - imported data having pageviews=0 and visitors=n should be bypassed", + %{conn: conn, site: site} do + site_import = + insert(:site_import, + site: site, + start_date: ~D[2005-01-01], + end_date: Timex.today(), + source: :universal_analytics + ) + + populate_stats(site, site_import.id, [ + build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]), + build(:pageview, + pathname: "/plausible.io", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:imported_pages, + page: "/skip-me", + date: ~D[2021-01-01], + visitors: 1, + pageviews: 0 + ), + build(:imported_pages, + page: "/include-me", + date: ~D[2021-01-01], + visitors: 1, + pageviews: 1 + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["pageviews"], + "date_range" => "all", + "dimensions" => ["event:page"], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["/"], "metrics" => [2]}, + %{"dimensions" => ["/plausible.io"], "metrics" => [1]}, + %{"dimensions" => ["/include-me"], "metrics" => [1]} + ] + end + + describe "breakdown by visit:exit_page with" do + setup %{site: site} do + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:pageview, + pathname: "/a", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + pathname: "/a", + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, + user_id: @user_id, + pathname: "/b", + timestamp: ~N[2021-01-01 00:35:00] + ), + build(:imported_exit_pages, + exit_page: "/b", + exits: 3, + visitors: 2, + pageviews: 5, + date: ~D[2021-01-01] + ) + ]) + end + + test "can query with visit:exit_page dimension", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visits"], + "date_range" => "all", + "dimensions" => ["visit:exit_page"], + "include" => %{"imports" => true} + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["/b"], "metrics" => [4]}, + %{"dimensions" => ["/a"], "metrics" => [1]} + ] + end + end + + describe "imported data" do + test "returns screen sizes breakdown when filtering by screen size", %{conn: conn, site: site} do + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:pageview, + timestamp: ~N[2021-01-01 00:00:01], + screen_size: "Mobile" + ), + build(:imported_devices, + device: "Mobile", + visitors: 3, + pageviews: 5, + date: ~D[2021-01-01] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "pageviews"], + "date_range" => "all", + "dimensions" => ["visit:device"], + "filters" => [ + ["is", "visit:device", ["Mobile"]] + ], + "include" => %{"imports" => true} + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [%{"dimensions" => ["Mobile"], "metrics" => [4, 6]}] + end + + test "returns custom event goals and pageview goals", %{conn: conn, site: site} do + insert(:goal, site: site, event_name: "Purchase") + insert(:goal, site: site, page_path: "/test") + + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:pageview, + timestamp: ~N[2021-01-01 00:00:01], + pathname: "/test" + ), + build(:event, + name: "Purchase", + timestamp: ~N[2021-01-01 00:00:03] + ), + build(:event, + name: "Purchase", + timestamp: ~N[2021-01-01 00:00:03] + ), + build(:imported_custom_events, + name: "Purchase", + visitors: 3, + events: 5, + date: ~D[2021-01-01] + ), + build(:imported_pages, + page: "/test", + visitors: 2, + pageviews: 2, + date: ~D[2021-01-01] + ), + build(:imported_visitors, visitors: 5, date: ~D[2021-01-01]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "dimensions" => ["event:goal"], + "metrics" => ["visitors", "events", "pageviews", "conversion_rate"], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Purchase"], "metrics" => [5, 7, 0, 62.5]}, + %{"dimensions" => ["Visit /test"], "metrics" => [3, 3, 3, 37.5]} + ] + end + + test "pageviews are returned as events for breakdown reports other than custom events", %{ + conn: conn, + site: site + } do + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:imported_browsers, browser: "Chrome", pageviews: 1, date: ~D[2021-01-01]), + build(:imported_devices, device: "Desktop", pageviews: 1, date: ~D[2021-01-01]), + build(:imported_entry_pages, entry_page: "/test", pageviews: 1, date: ~D[2021-01-01]), + build(:imported_exit_pages, exit_page: "/test", pageviews: 1, date: ~D[2021-01-01]), + build(:imported_locations, country: "EE", pageviews: 1, date: ~D[2021-01-01]), + build(:imported_operating_systems, + operating_system: "Mac", + pageviews: 1, + date: ~D[2021-01-01] + ), + build(:imported_pages, page: "/test", pageviews: 1, date: ~D[2021-01-01]), + build(:imported_sources, source: "Google", pageviews: 1, date: ~D[2021-01-01]) + ]) + + breakdown_and_first = fn dimension -> + conn + |> post("/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["events"], + "date_range" => ["2021-01-01", "2021-01-01"], + "dimensions" => [dimension], + "include" => %{"imports" => true} + }) + |> json_response(200) + |> Map.get("results") + |> List.first() + end + + assert %{"dimensions" => ["Chrome"], "metrics" => [1]} = + breakdown_and_first.("visit:browser") + + assert %{"dimensions" => ["Desktop"], "metrics" => [1]} = + breakdown_and_first.("visit:device") + + assert %{"dimensions" => ["EE"], "metrics" => [1]} = breakdown_and_first.("visit:country") + assert %{"dimensions" => ["Mac"], "metrics" => [1]} = breakdown_and_first.("visit:os") + assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("event:page") + + assert %{"dimensions" => ["Google"], "metrics" => [1]} = + breakdown_and_first.("visit:source") + end + + for goal_name <- Plausible.Imported.goals_with_url() do + test "returns url breakdown for #{goal_name} goal", %{conn: conn, site: site} do + insert(:goal, event_name: unquote(goal_name), site: site) + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:event, + name: unquote(goal_name), + "meta.key": ["url"], + "meta.value": ["https://one.com"] + ), + build(:imported_custom_events, + name: unquote(goal_name), + visitors: 2, + events: 5, + link_url: "https://one.com" + ), + build(:imported_custom_events, + name: unquote(goal_name), + visitors: 5, + events: 10, + link_url: "https://two.com" + ), + build(:imported_custom_events, + name: "some goal", + visitors: 5, + events: 10 + ), + build(:imported_visitors, visitors: 9) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "events", "conversion_rate"], + "date_range" => "all", + "dimensions" => ["event:props:url"], + "filters" => [ + ["is", "event:goal", [unquote(goal_name)]] + ], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["https://two.com"], "metrics" => [5, 10, 50]}, + %{"dimensions" => ["https://one.com"], "metrics" => [3, 6, 30]} + ] + + refute json_response(conn, 200)["meta"]["warning"] + end + end + + for goal_name <- Plausible.Imported.goals_with_path() do + test "returns path breakdown for #{goal_name} goal", %{conn: conn, site: site} do + insert(:goal, event_name: unquote(goal_name), site: site) + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:event, + name: unquote(goal_name), + "meta.key": ["path"], + "meta.value": ["/one"] + ), + build(:imported_custom_events, + name: unquote(goal_name), + visitors: 2, + events: 5, + path: "/one" + ), + build(:imported_custom_events, + name: unquote(goal_name), + visitors: 5, + events: 10, + path: "/two" + ), + build(:imported_custom_events, + name: "some goal", + visitors: 5, + events: 10 + ), + build(:imported_visitors, visitors: 9) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "events", "conversion_rate"], + "date_range" => "all", + "dimensions" => ["event:props:path"], + "filters" => [ + ["is", "event:goal", [unquote(goal_name)]] + ], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["/two"], "metrics" => [5, 10, 50]}, + %{"dimensions" => ["/one"], "metrics" => [3, 6, 30]} + ] + + refute json_response(conn, 200)["meta"]["warning"] + end + end + + test "adds a warning when query params are not supported for imported data", %{ + conn: conn, + site: site + } do + site_import = insert(:site_import, site: site) + + insert(:goal, event_name: "Signup", site: site) + + populate_stats(site, site_import.id, [ + build(:event, + name: "Signup", + "meta.key": ["package"], + "meta.value": ["large"] + ), + build(:imported_visitors, visitors: 9) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:package"], + "filters" => [ + ["is", "event:goal", ["Signup"]] + ], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["large"], "metrics" => [1]} + ] + + assert json_response(conn, 200)["meta"]["warning"] =~ + "Imported stats are not included in the results because query parameters are not supported." + end + + test "does not add a warning when there are no site imports", %{conn: conn, site: site} do + insert(:goal, event_name: "Signup", site: site) + + populate_stats(site, [ + build(:event, + name: "Signup", + "meta.key": ["package"], + "meta.value": ["large"] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:package"], + "filters" => [ + ["is", "event:goal", ["Signup"]] + ], + "include" => %{"imports" => true} + }) + + refute json_response(conn, 200)["meta"]["warning"] + end + + test "does not add a warning when import is out of queried date range", %{ + conn: conn, + site: site + } do + site_import = insert(:site_import, site: site, end_date: Date.add(Date.utc_today(), -3)) + + insert(:goal, event_name: "Signup", site: site) + + populate_stats(site, site_import.id, [ + build(:event, + name: "Signup", + "meta.key": ["package"], + "meta.value": ["large"] + ), + build(:imported_visitors, visitors: 9) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "day", + "dimensions" => ["event:props:package"], + "filters" => [ + ["is", "event:goal", ["Signup"]] + ], + "include" => %{"imports" => true} + }) + + refute json_response(conn, 200)["meta"]["warning"] + end + + test "applies multiple filters if the properties belong to the same table", %{ + conn: conn, + site: site + } do + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:imported_sources, source: "Google", utm_medium: "organic", utm_term: "one"), + build(:imported_sources, source: "Twitter", utm_medium: "organic", utm_term: "two"), + build(:imported_sources, + source: "Facebook", + utm_medium: "something_else", + utm_term: "one" + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "day", + "dimensions" => ["visit:source"], + "filters" => [ + ["is", "visit:utm_medium", ["organic"]], + ["is", "visit:utm_term", ["one"]] + ], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Google"], "metrics" => [1]} + ] + end + + test "ignores imported data if filtered property belongs to a different table than the breakdown property", + %{ + conn: conn, + site: site + } do + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:imported_sources, source: "Google"), + build(:imported_devices, device: "Desktop") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "day", + "dimensions" => ["visit:source"], + "filters" => [ + ["is", "visit:device", ["Desktop"]] + ], + "include" => %{"imports" => true} + }) + + assert %{ + "results" => [], + "meta" => meta + } = json_response(conn, 200) + + assert meta["warning"] =~ "Imported stats are not included in the results" + end + + test "imported country, region and city data", + %{ + conn: conn, + site: site + } do + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:pageview, + timestamp: ~N[2021-01-01 00:15:00], + country_code: "DE", + subdivision1_code: "DE-BE", + city_geoname_id: 2_950_159 + ), + build(:pageview, + timestamp: ~N[2021-01-01 00:15:00], + country_code: "DE", + subdivision1_code: "DE-BE", + city_geoname_id: 2_950_159 + ), + build(:pageview, + timestamp: ~N[2021-01-01 00:15:00], + country_code: "EE", + subdivision1_code: "EE-37", + city_geoname_id: 588_409 + ), + build(:imported_locations, country: "EE", region: "EE-37", city: 588_409, visitors: 33) + ]) + + for {dimension, stats_value, imports_value} <- [ + {"visit:country", "DE", "EE"}, + {"visit:region", "DE-BE", "EE-37"}, + {"visit:city", 2_950_159, 588_409} + ] do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => [dimension], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => [imports_value], "metrics" => [34]}, + %{"dimensions" => [stats_value], "metrics" => [2]} + ] + end + end + end +end diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_special_metrics_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_special_metrics_test.exs new file mode 100644 index 000000000..de403aa6b --- /dev/null +++ b/test/plausible_web/controllers/api/external_stats_controller/query_special_metrics_test.exs @@ -0,0 +1,193 @@ +defmodule PlausibleWeb.Api.ExternalStatsController.QuerySpecialMetricsTest do + use PlausibleWeb.ConnCase + + setup [:create_user, :create_new_site, :create_api_key, :use_api_key] + + test "returns conversion_rate in a goal filtered custom prop breakdown", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]), + build(:pageview, pathname: "/blog/2", "meta.key": ["author"], "meta.value": ["Uku"]), + build(:pageview, pathname: "/blog/3", "meta.key": ["author"], "meta.value": ["Uku"]), + build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Marko"]), + build(:pageview, + pathname: "/blog/2", + "meta.key": ["author"], + "meta.value": ["Marko"], + user_id: 1 + ), + build(:pageview, + pathname: "/blog/3", + "meta.key": ["author"], + "meta.value": ["Marko"], + user_id: 1 + ), + build(:pageview, pathname: "/blog"), + build(:pageview, "meta.key": ["author"], "meta.value": ["Marko"]), + build(:pageview) + ]) + + insert(:goal, %{site: site, page_path: "/blog**"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "filters" => [["matches", "event:goal", ["Visit /blog**"]]], + "metrics" => ["visitors", "events", "conversion_rate"], + "dimensions" => ["event:props:author"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Uku"], "metrics" => [3, 3, 37.5]}, + %{"dimensions" => ["Marko"], "metrics" => [2, 3, 25.0]}, + %{"dimensions" => ["(none)"], "metrics" => [1, 1, 12.5]} + ] + end + + test "returns conversion_rate alone in a goal filtered custom prop breakdown", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]), + build(:pageview) + ]) + + insert(:goal, %{site: site, page_path: "/blog**"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["conversion_rate"], + "date_range" => "all", + "dimensions" => ["event:props:author"], + "filters" => [["matches", "event:goal", ["Visit /blog**"]]] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Uku"], "metrics" => [50]} + ] + end + + test "returns conversion_rate in a goal filtered event:page breakdown", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, pathname: "/en/register", name: "pageview"), + build(:event, pathname: "/en/register", name: "Signup"), + build(:event, pathname: "/en/register", name: "Signup"), + build(:event, pathname: "/it/register", name: "Signup", user_id: 1), + build(:event, pathname: "/it/register", name: "Signup", user_id: 1), + build(:event, pathname: "/it/register", name: "pageview") + ]) + + insert(:goal, %{site: site, event_name: "Signup"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [["is", "event:goal", ["Signup"]]], + "metrics" => ["visitors", "events", "group_conversion_rate"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["/en/register"], "metrics" => [2, 2, 66.7]}, + %{"dimensions" => ["/it/register"], "metrics" => [1, 2, 50.0]} + ] + end + + test "returns conversion_rate alone in a goal filtered event:page breakdown", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, pathname: "/en/register", name: "pageview"), + build(:event, pathname: "/en/register", name: "Signup") + ]) + + insert(:goal, %{site: site, event_name: "Signup"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "filters" => [["is", "event:goal", ["Signup"]]], + "metrics" => ["group_conversion_rate"], + "dimensions" => ["event:page"] + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["/en/register"], "metrics" => [50.0]} + ] + end + + test "returns conversion_rate in a multi-goal filtered visit:screen_size breakdown", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, screen_size: "Mobile", name: "pageview"), + build(:event, screen_size: "Mobile", name: "AddToCart"), + build(:event, screen_size: "Mobile", name: "AddToCart"), + build(:event, screen_size: "Desktop", name: "AddToCart", user_id: 1), + build(:event, screen_size: "Desktop", name: "Purchase", user_id: 1), + build(:event, screen_size: "Desktop", name: "pageview") + ]) + + # Make sure that revenue goals are treated the same + # way as regular custom event goals + insert(:goal, %{site: site, event_name: "Purchase", currency: :EUR}) + insert(:goal, %{site: site, event_name: "AddToCart"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "events", "group_conversion_rate"], + "date_range" => "all", + "dimensions" => ["visit:device"], + "filters" => [["is", "event:goal", ["AddToCart", "Purchase"]]] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Mobile"], "metrics" => [2, 2, 66.7]}, + %{"dimensions" => ["Desktop"], "metrics" => [1, 2, 50]} + ] + end + + test "returns conversion_rate alone in a goal filtered visit:screen_size breakdown", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, screen_size: "Mobile", name: "pageview"), + build(:event, screen_size: "Mobile", name: "AddToCart") + ]) + + insert(:goal, %{site: site, event_name: "AddToCart"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["conversion_rate"], + "date_range" => "all", + "dimensions" => ["visit:device"], + "filters" => [["is", "event:goal", ["AddToCart"]]] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Mobile"], "metrics" => [50]} + ] + end +end diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs index 945419639..8bcd82a1a 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs @@ -1,239 +1,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do use PlausibleWeb.ConnCase - alias Plausible.Billing.Feature @user_id 1231 setup [:create_user, :create_new_site, :create_api_key, :use_api_key] - describe "feature access" do - test "cannot break down by a custom prop without access to the props feature", %{ - conn: conn, - user: user, - site: site - } do - ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id) - insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "dimensions" => ["event:props:author"] - }) - - assert json_response(conn, 400)["error"] == - "The owner of this site does not have access to the custom properties feature" - end - - test "can break down by an internal prop key without access to the props feature", %{ - conn: conn, - user: user, - site: site - } do - ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id) - insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "dimensions" => ["event:props:path"] - }) - - assert json_response(conn, 200)["results"] - end - - test "cannot filter by a custom prop without access to the props feature", %{ - conn: conn, - user: user, - site: site - } do - ep = - insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id) - - insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "dimensions" => ["visit:source"], - "filters" => [["is", "event:props:author", ["Uku"]]] - }) - - assert json_response(conn, 400)["error"] == - "The owner of this site does not have access to the custom properties feature" - end - - test "can filter by an internal prop key without access to the props feature", %{ - conn: conn, - user: user, - site: site - } do - ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id) - insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "dimensions" => ["visit:source"], - "filters" => [["is", "event:props:url", ["whatever"]]] - }) - - assert json_response(conn, 200)["results"] - end - end - - describe "param validation" do - test "does not allow querying conversion_rate without a goal filter", %{ - conn: conn, - site: site - } do - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["conversion_rate"], - "date_range" => "all", - "dimensions" => ["event:page"], - "filters" => [["is", "event:props:author", ["Uku"]]] - }) - - assert json_response(conn, 400)["error"] == - "Metric `conversion_rate` can only be queried with event:goal filters or dimensions" - end - - test "validates that dimensions are valid", %{conn: conn, site: site} do - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "dimensions" => ["badproperty"] - }) - - assert json_response(conn, 400)["error"] =~ "Invalid dimensions" - end - - test "empty custom property is invalid", %{conn: conn, site: site} do - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "dimensions" => ["event:props:"] - }) - - assert json_response(conn, 400)["error"] =~ "Invalid dimensions" - end - - test "validates that correct date range is used", %{conn: conn, site: site} do - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "bad_period", - "dimensions" => ["event:name"] - }) - - assert json_response(conn, 400)["error"] =~ "Invalid date_range" - end - - test "fails when an invalid metric is provided", %{conn: conn, site: site} do - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visitors", "baa"], - "date_range" => "all", - "dimensions" => ["event:name"] - }) - - assert json_response(conn, 400)["error"] =~ "Unknown metric '\"baa\"'" - end - - test "session metrics cannot be used with event:name dimension", %{conn: conn, site: site} do - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visitors", "bounce_rate"], - "date_range" => "all", - "dimensions" => ["event:name"] - }) - - assert json_response(conn, 400)["error"] =~ - "Session metric(s) `bounce_rate` cannot be queried along with event dimensions" - end - - test "session metrics cannot be used with event:props:* dimension", %{conn: conn, site: site} do - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visitors", "bounce_rate"], - "date_range" => "all", - "dimensions" => ["event:props:url"] - }) - - assert json_response(conn, 400)["error"] =~ - "Session metric(s) `bounce_rate` cannot be queried along with event dimensions" - end - - test "validates that metric views_per_visit cannot be used with event:page filter", %{ - conn: conn, - site: site - } do - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "date_range" => "all", - "metrics" => ["views_per_visit"], - "filters" => [["is", "event:page", ["/something"]]] - }) - - assert json_response(conn, 400) == %{ - "error" => - "Metric `views_per_visit` cannot be queried with a filter on `event:page`" - } - end - - test "validates that metric views_per_visit cannot be used together with dimensions", %{ - conn: conn, - site: site - } do - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "date_range" => "all", - "metrics" => ["views_per_visit"], - "dimensions" => ["event:name"] - }) - - assert json_response(conn, 400) == %{ - "error" => "Metric `views_per_visit` cannot be queried with `dimensions`" - } - end - - test "validates a metric can't be asked multiple times", %{ - conn: conn, - site: site - } do - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "date_range" => "all", - "metrics" => ["views_per_visit", "visitors", "visitors"] - }) - - assert json_response(conn, 400) == %{ - "error" => "Metrics cannot be queried multiple times" - } - end - end - test "aggregates a single metric", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]), @@ -1065,36 +836,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do end end - describe "aggregation with imported data" do - setup :create_site_import - - test "does not count imported stats unless specified", %{ - conn: conn, - site: site, - site_import: site_import - } do - populate_stats(site, site_import.id, [ - build(:imported_visitors, date: ~D[2023-01-01]), - build(:pageview, timestamp: ~N[2023-01-01 00:00:00]) - ]) - - query_params = %{ - "site_id" => site.domain, - "date_range" => "all", - "metrics" => ["pageviews"] - } - - conn1 = post(conn, "/api/v2/query", query_params) - - assert json_response(conn1, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}] - - conn2 = post(conn, "/api/v2/query", Map.put(query_params, "include", %{"imports" => true})) - - assert json_response(conn2, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}] - refute json_response(conn2, 200)["meta"]["warning"] - end - end - describe "timeseries" do test "shows hourly data for a certain date with time_labels", %{conn: conn, site: site} do populate_stats(site, [ @@ -1405,63 +1146,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do ] end - test "breaks down all metrics by visit:referrer with imported data", %{conn: conn, site: site} do - site_import = - insert(:site_import, - site: site, - start_date: ~D[2005-01-01], - end_date: Timex.today(), - source: :universal_analytics - ) - - populate_stats(site, site_import.id, [ - build(:pageview, referrer: "site.com", timestamp: ~N[2021-01-01 00:00:00]), - build(:pageview, referrer: "site.com/1", timestamp: ~N[2021-01-01 00:00:00]), - build(:imported_sources, - referrer: "site.com", - date: ~D[2021-01-01], - visitors: 2, - visits: 2, - pageviews: 2, - bounces: 1, - visit_duration: 120 - ), - build(:imported_sources, - referrer: "site.com/2", - date: ~D[2021-01-01], - visitors: 2, - visits: 2, - pageviews: 2, - bounces: 2, - visit_duration: 0 - ), - build(:imported_sources, - date: ~D[2021-01-01], - visitors: 10, - visits: 11, - pageviews: 50, - bounces: 0, - visit_duration: 1100 - ) - ]) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visitors", "visits", "pageviews", "bounce_rate", "visit_duration"], - "date_range" => "all", - "dimensions" => ["visit:referrer"], - "include" => %{"imports" => true} - }) - - assert json_response(conn, 200)["results"] == [ - %{"dimensions" => ["Direct / None"], "metrics" => [10, 11, 50, 0.0, 100.0]}, - %{"dimensions" => ["site.com"], "metrics" => [3, 3, 3, 67.0, 40.0]}, - %{"dimensions" => ["site.com/2"], "metrics" => [2, 2, 2, 100.0, 0.0]}, - %{"dimensions" => ["site.com/1"], "metrics" => [1, 1, 1, 100.0, 0.0]} - ] - end - for {dimension, attr} <- [ {"visit:utm_campaign", :utm_campaign}, {"visit:utm_source", :utm_source}, @@ -1657,111 +1341,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do ] end - test "breaks down all metrics by visit:utm_source with imported data", %{conn: conn, site: site} do - site_import = - insert(:site_import, - site: site, - start_date: ~D[2005-01-01], - end_date: Timex.today(), - source: :universal_analytics - ) - - populate_stats(site, site_import.id, [ - build(:pageview, utm_source: "SomeUTMSource", timestamp: ~N[2021-01-01 00:00:00]), - build(:pageview, utm_source: "SomeUTMSource-1", timestamp: ~N[2021-01-01 00:00:00]), - build(:imported_sources, - utm_source: "SomeUTMSource", - date: ~D[2021-01-01], - visitors: 2, - visits: 2, - pageviews: 2, - bounces: 1, - visit_duration: 120 - ), - build(:imported_sources, - utm_source: "SomeUTMSource-2", - date: ~D[2021-01-01], - visitors: 2, - visits: 2, - pageviews: 2, - bounces: 2, - visit_duration: 0 - ), - build(:imported_sources, - date: ~D[2021-01-01], - visitors: 10, - visits: 11, - pageviews: 50, - bounces: 0, - visit_duration: 1100 - ) - ]) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visitors", "visits", "pageviews", "bounce_rate", "visit_duration"], - "date_range" => "all", - "dimensions" => ["visit:utm_source"], - "include" => %{"imports" => true} - }) - - %{"results" => results} = json_response(conn, 200) - - assert results == [ - %{"dimensions" => ["SomeUTMSource"], "metrics" => [3, 3, 3, 67.0, 40.0]}, - %{"dimensions" => ["SomeUTMSource-2"], "metrics" => [2, 2, 2, 100.0, 0.0]}, - %{"dimensions" => ["SomeUTMSource-1"], "metrics" => [1, 1, 1, 100.0, 0.0]} - ] - end - - test "pageviews breakdown by event:page - imported data having pageviews=0 and visitors=n should be bypassed", - %{conn: conn, site: site} do - site_import = - insert(:site_import, - site: site, - start_date: ~D[2005-01-01], - end_date: Timex.today(), - source: :universal_analytics - ) - - populate_stats(site, site_import.id, [ - build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]), - build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]), - build(:pageview, - pathname: "/plausible.io", - timestamp: ~N[2021-01-01 00:00:00] - ), - build(:imported_pages, - page: "/skip-me", - date: ~D[2021-01-01], - visitors: 1, - pageviews: 0 - ), - build(:imported_pages, - page: "/include-me", - date: ~D[2021-01-01], - visitors: 1, - pageviews: 1 - ) - ]) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["pageviews"], - "date_range" => "all", - "dimensions" => ["event:page"], - "include" => %{"imports" => true} - }) - - assert json_response(conn, 200)["results"] == [ - %{"dimensions" => ["/"], "metrics" => [2]}, - %{"dimensions" => ["/plausible.io"], "metrics" => [1]}, - %{"dimensions" => ["/include-me"], "metrics" => [1]} - ] - end - test "breakdown by event:page", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]), @@ -1811,54 +1390,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do ] end - describe "breakdown by visit:exit_page" do - setup %{site: site} do - site_import = insert(:site_import, site: site) - - populate_stats(site, site_import.id, [ - build(:pageview, - pathname: "/a", - timestamp: ~N[2021-01-01 00:00:00] - ), - build(:pageview, - user_id: @user_id, - pathname: "/a", - timestamp: ~N[2021-01-01 00:25:00] - ), - build(:pageview, - user_id: @user_id, - pathname: "/b", - timestamp: ~N[2021-01-01 00:35:00] - ), - build(:imported_exit_pages, - exit_page: "/b", - exits: 3, - visitors: 2, - pageviews: 5, - date: ~D[2021-01-01] - ) - ]) - end - - test "can query with visit:exit_page dimension", %{conn: conn, site: site} do - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visits"], - "date_range" => "all", - "dimensions" => ["visit:exit_page"], - "include" => %{"imports" => true} - }) - - %{"results" => results} = json_response(conn, 200) - - assert results == [ - %{"dimensions" => ["/b"], "metrics" => [4]}, - %{"dimensions" => ["/a"], "metrics" => [1]} - ] - end - end - describe "custom events" do test "can breakdown by event:name", %{conn: conn, site: site} do populate_stats(site, [ @@ -2248,8 +1779,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do "site_id" => site.domain, "metrics" => ["pageviews"], "date_range" => "all", - "dimensions" => ["event:props:package"], - "filters" => [] + "dimensions" => ["event:props:package"] }) %{"results" => results} = json_response(conn, 200) @@ -2317,85 +1847,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do end end - describe "breakdown by event:goal" do - test "returns custom event goals and pageview goals", %{conn: conn, site: site} do - insert(:goal, %{site: site, event_name: "Purchase"}) - insert(:goal, %{site: site, page_path: "/test"}) - - populate_stats(site, [ - build(:pageview, - timestamp: ~N[2021-01-01 00:00:01], - pathname: "/test" - ), - build(:event, - name: "Purchase", - timestamp: ~N[2021-01-01 00:00:03] - ), - build(:event, - name: "Purchase", - timestamp: ~N[2021-01-01 00:00:03] - ) - ]) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "date_range" => "all", - "metrics" => ["visitors"], - "dimensions" => ["event:goal"] - }) - - assert json_response(conn, 200)["results"] == [ - %{"dimensions" => ["Purchase"], "metrics" => [2]}, - %{"dimensions" => ["Visit /test"], "metrics" => [1]} - ] - end - - test "returns pageview goals containing wildcards", %{conn: conn, site: site} do - insert(:goal, %{site: site, page_path: "/**/post"}) - insert(:goal, %{site: site, page_path: "/blog**"}) - - populate_stats(site, [ - build(:pageview, pathname: "/blog", user_id: @user_id), - build(:pageview, pathname: "/blog/post-1", user_id: @user_id), - build(:pageview, pathname: "/blog/post-2", user_id: @user_id), - build(:pageview, pathname: "/blog/something/post"), - build(:pageview, pathname: "/different/page/post") - ]) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "date_range" => "all", - "metrics" => ["visitors", "pageviews"], - "dimensions" => ["event:goal"], - "order_by" => [["pageviews", "desc"]] - }) - - assert json_response(conn, 200)["results"] == [ - %{"dimensions" => ["Visit /blog**"], "metrics" => [2, 4]}, - %{"dimensions" => ["Visit /**/post"], "metrics" => [2, 2]} - ] - end - - test "does not return goals that are not configured for the site", %{conn: conn, site: site} do - populate_stats(site, [ - build(:pageview, pathname: "/register"), - build(:event, name: "Signup") - ]) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "date_range" => "all", - "metrics" => ["visitors", "pageviews"], - "dimensions" => ["event:goal"] - }) - - assert json_response(conn, 200)["results"] == [] - end - end - test "event:goal filter returns 400 when goal not configured", %{conn: conn, site: site} do conn = post(conn, "/api/v2/query", %{ @@ -2802,21 +2253,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do ) ]) - conn = - get(conn, "/api/v1/stats/breakdown", %{ - "site_id" => site.domain, - "period" => "day", - "property" => "event:page", - "filters" => "event:goal == Register" - }) - - assert json_response(conn, 200) == %{ - "results" => [ - %{"page" => "/en/register", "visitors" => 2}, - %{"page" => "/it/register", "visitors" => 1} - ] - } - conn = post(conn, "/api/v2/query", %{ "site_id" => site.domain, @@ -3159,241 +2595,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do end describe "metrics" do - test "returns conversion_rate in an event:goal breakdown", %{conn: conn, site: site} do - populate_stats(site, [ - build(:event, name: "Signup", user_id: 1), - build(:event, name: "Signup", user_id: 1), - build(:pageview, pathname: "/blog"), - build(:pageview, pathname: "/blog/post"), - build(:pageview) - ]) - - insert(:goal, %{site: site, event_name: "Signup"}) - insert(:goal, %{site: site, page_path: "/blog**"}) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "date_range" => "all", - "metrics" => ["visitors", "events", "conversion_rate"], - "dimensions" => ["event:goal"] - }) - - assert json_response(conn, 200)["results"] == [ - %{"dimensions" => ["Visit /blog**"], "metrics" => [2, 2, 50.0]}, - %{"dimensions" => ["Signup"], "metrics" => [1, 2, 25.0]} - ] - end - - test "returns conversion_rate alone in an event:goal breakdown", %{conn: conn, site: site} do - populate_stats(site, [ - build(:event, name: "Signup", user_id: 1), - build(:pageview) - ]) - - insert(:goal, %{site: site, event_name: "Signup"}) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "date_range" => "all", - "metrics" => ["conversion_rate"], - "dimensions" => ["event:goal"] - }) - - assert json_response(conn, 200)["results"] == [ - %{"dimensions" => ["Signup"], "metrics" => [50.0]} - ] - end - - test "returns conversion_rate in a goal filtered custom prop breakdown", %{ - conn: conn, - site: site - } do - populate_stats(site, [ - build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]), - build(:pageview, pathname: "/blog/2", "meta.key": ["author"], "meta.value": ["Uku"]), - build(:pageview, pathname: "/blog/3", "meta.key": ["author"], "meta.value": ["Uku"]), - build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Marko"]), - build(:pageview, - pathname: "/blog/2", - "meta.key": ["author"], - "meta.value": ["Marko"], - user_id: 1 - ), - build(:pageview, - pathname: "/blog/3", - "meta.key": ["author"], - "meta.value": ["Marko"], - user_id: 1 - ), - build(:pageview, pathname: "/blog"), - build(:pageview, "meta.key": ["author"], "meta.value": ["Marko"]), - build(:pageview) - ]) - - insert(:goal, %{site: site, page_path: "/blog**"}) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "date_range" => "all", - "filters" => [["matches", "event:goal", ["Visit /blog**"]]], - "metrics" => ["visitors", "events", "conversion_rate"], - "dimensions" => ["event:props:author"] - }) - - assert json_response(conn, 200)["results"] == [ - %{"dimensions" => ["Uku"], "metrics" => [3, 3, 37.5]}, - %{"dimensions" => ["Marko"], "metrics" => [2, 3, 25.0]}, - %{"dimensions" => ["(none)"], "metrics" => [1, 1, 12.5]} - ] - end - - test "returns conversion_rate alone in a goal filtered custom prop breakdown", %{ - conn: conn, - site: site - } do - populate_stats(site, [ - build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]), - build(:pageview) - ]) - - insert(:goal, %{site: site, page_path: "/blog**"}) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["conversion_rate"], - "date_range" => "all", - "dimensions" => ["event:props:author"], - "filters" => [["matches", "event:goal", ["Visit /blog**"]]] - }) - - %{"results" => results} = json_response(conn, 200) - - assert results == [ - %{"dimensions" => ["Uku"], "metrics" => [50]} - ] - end - - test "returns conversion_rate in a goal filtered event:page breakdown", %{ - conn: conn, - site: site - } do - populate_stats(site, [ - build(:event, pathname: "/en/register", name: "pageview"), - build(:event, pathname: "/en/register", name: "Signup"), - build(:event, pathname: "/en/register", name: "Signup"), - build(:event, pathname: "/it/register", name: "Signup", user_id: 1), - build(:event, pathname: "/it/register", name: "Signup", user_id: 1), - build(:event, pathname: "/it/register", name: "pageview") - ]) - - insert(:goal, %{site: site, event_name: "Signup"}) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "date_range" => "all", - "dimensions" => ["event:page"], - "filters" => [["is", "event:goal", ["Signup"]]], - "metrics" => ["visitors", "events", "group_conversion_rate"] - }) - - assert json_response(conn, 200)["results"] == [ - %{"dimensions" => ["/en/register"], "metrics" => [2, 2, 66.7]}, - %{"dimensions" => ["/it/register"], "metrics" => [1, 2, 50.0]} - ] - end - - test "returns conversion_rate alone in a goal filtered event:page breakdown", %{ - conn: conn, - site: site - } do - populate_stats(site, [ - build(:event, pathname: "/en/register", name: "pageview"), - build(:event, pathname: "/en/register", name: "Signup") - ]) - - insert(:goal, %{site: site, event_name: "Signup"}) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "date_range" => "all", - "filters" => [["is", "event:goal", ["Signup"]]], - "metrics" => ["group_conversion_rate"], - "dimensions" => ["event:page"] - }) - - assert json_response(conn, 200)["results"] == [ - %{"dimensions" => ["/en/register"], "metrics" => [50.0]} - ] - end - - test "returns conversion_rate in a multi-goal filtered visit:screen_size breakdown", %{ - conn: conn, - site: site - } do - populate_stats(site, [ - build(:event, screen_size: "Mobile", name: "pageview"), - build(:event, screen_size: "Mobile", name: "AddToCart"), - build(:event, screen_size: "Mobile", name: "AddToCart"), - build(:event, screen_size: "Desktop", name: "AddToCart", user_id: 1), - build(:event, screen_size: "Desktop", name: "Purchase", user_id: 1), - build(:event, screen_size: "Desktop", name: "pageview") - ]) - - # Make sure that revenue goals are treated the same - # way as regular custom event goals - insert(:goal, %{site: site, event_name: "Purchase", currency: :EUR}) - insert(:goal, %{site: site, event_name: "AddToCart"}) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visitors", "events", "group_conversion_rate"], - "date_range" => "all", - "dimensions" => ["visit:device"], - "filters" => [["is", "event:goal", ["AddToCart", "Purchase"]]] - }) - - %{"results" => results} = json_response(conn, 200) - - assert results == [ - %{"dimensions" => ["Mobile"], "metrics" => [2, 2, 66.7]}, - %{"dimensions" => ["Desktop"], "metrics" => [1, 2, 50]} - ] - end - - test "returns conversion_rate alone in a goal filtered visit:screen_size breakdown", %{ - conn: conn, - site: site - } do - populate_stats(site, [ - build(:event, screen_size: "Mobile", name: "pageview"), - build(:event, screen_size: "Mobile", name: "AddToCart") - ]) - - insert(:goal, %{site: site, event_name: "AddToCart"}) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["conversion_rate"], - "date_range" => "all", - "dimensions" => ["visit:device"], - "filters" => [["is", "event:goal", ["AddToCart"]]] - }) - - %{"results" => results} = json_response(conn, 200) - - assert results == [ - %{"dimensions" => ["Mobile"], "metrics" => [50]} - ] - end - test "all metrics for breakdown by visit prop", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, @@ -3434,8 +2635,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do "visit_duration" ], "date_range" => "all", - "dimensions" => ["visit:source"], - "filters" => [] + "dimensions" => ["visit:source"] }) %{"results" => results} = json_response(conn, 200) @@ -3483,58 +2683,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do ] end - test "filter by custom event property", %{conn: conn, site: site} do - populate_stats(site, [ - build(:event, - name: "Purchase", - "meta.key": ["package"], - "meta.value": ["business"], - browser: "Chrome", - timestamp: ~N[2021-01-01 00:00:00] - ), - build(:event, - name: "Purchase", - "meta.key": ["package"], - "meta.value": ["business"], - browser: "Safari", - timestamp: ~N[2021-01-01 00:00:00] - ), - build(:event, - name: "Purchase", - "meta.key": ["package"], - "meta.value": ["business"], - browser: "Safari", - timestamp: ~N[2021-01-01 00:25:00] - ), - build(:event, - name: "Purchase", - "meta.key": ["package"], - "meta.value": ["personal"], - browser: "IE", - timestamp: ~N[2021-01-01 00:25:00] - ) - ]) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "dimensions" => ["visit:browser"], - "filters" => [ - ["is", "event:name", ["Purchase"]], - ["is", "event:props:package", ["business"]] - ] - }) - - %{"results" => results} = json_response(conn, 200) - - assert results == [ - %{"dimensions" => ["Safari"], "metrics" => [2]}, - %{"dimensions" => ["Chrome"], "metrics" => [1]} - ] - end - test "all metrics for breakdown by event prop", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, @@ -3578,452 +2726,56 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do end end - describe "imported data" do - test "returns screen sizes breakdown when filtering by screen size", %{conn: conn, site: site} do - site_import = insert(:site_import, site: site) + test "filtering by custom event property", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Purchase", + "meta.key": ["package"], + "meta.value": ["business"], + browser: "Chrome", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Purchase", + "meta.key": ["package"], + "meta.value": ["business"], + browser: "Safari", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Purchase", + "meta.key": ["package"], + "meta.value": ["business"], + browser: "Safari", + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:event, + name: "Purchase", + "meta.key": ["package"], + "meta.value": ["personal"], + browser: "IE", + timestamp: ~N[2021-01-01 00:25:00] + ) + ]) - populate_stats(site, site_import.id, [ - build(:pageview, - timestamp: ~N[2021-01-01 00:00:01], - screen_size: "Mobile" - ), - build(:imported_devices, - device: "Mobile", - visitors: 3, - pageviews: 5, - date: ~D[2021-01-01] - ) - ]) + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:browser"], + "filters" => [ + ["is", "event:name", ["Purchase"]], + ["is", "event:props:package", ["business"]] + ] + }) - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visitors", "pageviews"], - "date_range" => "all", - "dimensions" => ["visit:device"], - "filters" => [ - ["is", "visit:device", ["Mobile"]] - ], - "include" => %{"imports" => true} - }) + %{"results" => results} = json_response(conn, 200) - %{"results" => results} = json_response(conn, 200) - - assert results == [%{"dimensions" => ["Mobile"], "metrics" => [4, 6]}] - end - - test "returns custom event goals and pageview goals", %{conn: conn, site: site} do - insert(:goal, site: site, event_name: "Purchase") - insert(:goal, site: site, page_path: "/test") - - site_import = insert(:site_import, site: site) - - populate_stats(site, site_import.id, [ - build(:pageview, - timestamp: ~N[2021-01-01 00:00:01], - pathname: "/test" - ), - build(:event, - name: "Purchase", - timestamp: ~N[2021-01-01 00:00:03] - ), - build(:event, - name: "Purchase", - timestamp: ~N[2021-01-01 00:00:03] - ), - build(:imported_custom_events, - name: "Purchase", - visitors: 3, - events: 5, - date: ~D[2021-01-01] - ), - build(:imported_pages, - page: "/test", - visitors: 2, - pageviews: 2, - date: ~D[2021-01-01] - ), - build(:imported_visitors, visitors: 5, date: ~D[2021-01-01]) - ]) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "date_range" => "all", - "dimensions" => ["event:goal"], - "metrics" => ["visitors", "events", "pageviews", "conversion_rate"], - "include" => %{"imports" => true} - }) - - assert json_response(conn, 200)["results"] == [ - %{"dimensions" => ["Purchase"], "metrics" => [5, 7, 0, 62.5]}, - %{"dimensions" => ["Visit /test"], "metrics" => [3, 3, 3, 37.5]} - ] - end - - test "pageviews are returned as events for breakdown reports other than custom events", %{ - conn: conn, - site: site - } do - site_import = insert(:site_import, site: site) - - populate_stats(site, site_import.id, [ - build(:imported_browsers, browser: "Chrome", pageviews: 1, date: ~D[2021-01-01]), - build(:imported_devices, device: "Desktop", pageviews: 1, date: ~D[2021-01-01]), - build(:imported_entry_pages, entry_page: "/test", pageviews: 1, date: ~D[2021-01-01]), - build(:imported_exit_pages, exit_page: "/test", pageviews: 1, date: ~D[2021-01-01]), - build(:imported_locations, country: "EE", pageviews: 1, date: ~D[2021-01-01]), - build(:imported_operating_systems, - operating_system: "Mac", - pageviews: 1, - date: ~D[2021-01-01] - ), - build(:imported_pages, page: "/test", pageviews: 1, date: ~D[2021-01-01]), - build(:imported_sources, source: "Google", pageviews: 1, date: ~D[2021-01-01]) - ]) - - breakdown_and_first = fn dimension -> - conn - |> post("/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["events"], - "date_range" => ["2021-01-01", "2021-01-01"], - "dimensions" => [dimension], - "include" => %{"imports" => true} - }) - |> json_response(200) - |> Map.get("results") - |> List.first() - end - - assert %{"dimensions" => ["Chrome"], "metrics" => [1]} = - breakdown_and_first.("visit:browser") - - assert %{"dimensions" => ["Desktop"], "metrics" => [1]} = - breakdown_and_first.("visit:device") - - assert %{"dimensions" => ["EE"], "metrics" => [1]} = breakdown_and_first.("visit:country") - assert %{"dimensions" => ["Mac"], "metrics" => [1]} = breakdown_and_first.("visit:os") - assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("event:page") - - assert %{"dimensions" => ["Google"], "metrics" => [1]} = - breakdown_and_first.("visit:source") - end - - for goal_name <- Plausible.Imported.goals_with_url() do - test "returns url breakdown for #{goal_name} goal", %{conn: conn, site: site} do - insert(:goal, event_name: unquote(goal_name), site: site) - site_import = insert(:site_import, site: site) - - populate_stats(site, site_import.id, [ - build(:event, - name: unquote(goal_name), - "meta.key": ["url"], - "meta.value": ["https://one.com"] - ), - build(:imported_custom_events, - name: unquote(goal_name), - visitors: 2, - events: 5, - link_url: "https://one.com" - ), - build(:imported_custom_events, - name: unquote(goal_name), - visitors: 5, - events: 10, - link_url: "https://two.com" - ), - build(:imported_custom_events, - name: "some goal", - visitors: 5, - events: 10 - ), - build(:imported_visitors, visitors: 9) - ]) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visitors", "events", "conversion_rate"], - "date_range" => "all", - "dimensions" => ["event:props:url"], - "filters" => [ - ["is", "event:goal", [unquote(goal_name)]] - ], - "include" => %{"imports" => true} - }) - - assert json_response(conn, 200)["results"] == [ - %{"dimensions" => ["https://two.com"], "metrics" => [5, 10, 50]}, - %{"dimensions" => ["https://one.com"], "metrics" => [3, 6, 30]} - ] - - refute json_response(conn, 200)["meta"]["warning"] - end - end - - for goal_name <- Plausible.Imported.goals_with_path() do - test "returns path breakdown for #{goal_name} goal", %{conn: conn, site: site} do - insert(:goal, event_name: unquote(goal_name), site: site) - site_import = insert(:site_import, site: site) - - populate_stats(site, site_import.id, [ - build(:event, - name: unquote(goal_name), - "meta.key": ["path"], - "meta.value": ["/one"] - ), - build(:imported_custom_events, - name: unquote(goal_name), - visitors: 2, - events: 5, - path: "/one" - ), - build(:imported_custom_events, - name: unquote(goal_name), - visitors: 5, - events: 10, - path: "/two" - ), - build(:imported_custom_events, - name: "some goal", - visitors: 5, - events: 10 - ), - build(:imported_visitors, visitors: 9) - ]) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visitors", "events", "conversion_rate"], - "date_range" => "all", - "dimensions" => ["event:props:path"], - "filters" => [ - ["is", "event:goal", [unquote(goal_name)]] - ], - "include" => %{"imports" => true} - }) - - assert json_response(conn, 200)["results"] == [ - %{"dimensions" => ["/two"], "metrics" => [5, 10, 50]}, - %{"dimensions" => ["/one"], "metrics" => [3, 6, 30]} - ] - - refute json_response(conn, 200)["meta"]["warning"] - end - end - - test "adds a warning when query params are not supported for imported data", %{ - conn: conn, - site: site - } do - site_import = insert(:site_import, site: site) - - insert(:goal, event_name: "Signup", site: site) - - populate_stats(site, site_import.id, [ - build(:event, - name: "Signup", - "meta.key": ["package"], - "meta.value": ["large"] - ), - build(:imported_visitors, visitors: 9) - ]) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "dimensions" => ["event:props:package"], - "filters" => [ - ["is", "event:goal", ["Signup"]] - ], - "include" => %{"imports" => true} - }) - - assert json_response(conn, 200)["results"] == [ - %{"dimensions" => ["large"], "metrics" => [1]} - ] - - assert json_response(conn, 200)["meta"]["warning"] =~ - "Imported stats are not included in the results because query parameters are not supported." - end - - test "does not add a warning when there are no site imports", %{conn: conn, site: site} do - insert(:goal, event_name: "Signup", site: site) - - populate_stats(site, [ - build(:event, - name: "Signup", - "meta.key": ["package"], - "meta.value": ["large"] - ) - ]) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "dimensions" => ["event:props:package"], - "filters" => [ - ["is", "event:goal", ["Signup"]] - ], - "include" => %{"imports" => true} - }) - - refute json_response(conn, 200)["meta"]["warning"] - end - - test "does not add a warning when import is out of queried date range", %{ - conn: conn, - site: site - } do - site_import = insert(:site_import, site: site, end_date: Date.add(Date.utc_today(), -3)) - - insert(:goal, event_name: "Signup", site: site) - - populate_stats(site, site_import.id, [ - build(:event, - name: "Signup", - "meta.key": ["package"], - "meta.value": ["large"] - ), - build(:imported_visitors, visitors: 9) - ]) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "day", - "dimensions" => ["event:props:package"], - "filters" => [ - ["is", "event:goal", ["Signup"]] - ], - "include" => %{"imports" => true} - }) - - refute json_response(conn, 200)["meta"]["warning"] - end - - test "applies multiple filters if the properties belong to the same table", %{ - conn: conn, - site: site - } do - site_import = insert(:site_import, site: site) - - populate_stats(site, site_import.id, [ - build(:imported_sources, source: "Google", utm_medium: "organic", utm_term: "one"), - build(:imported_sources, source: "Twitter", utm_medium: "organic", utm_term: "two"), - build(:imported_sources, - source: "Facebook", - utm_medium: "something_else", - utm_term: "one" - ) - ]) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "day", - "dimensions" => ["visit:source"], - "filters" => [ - ["is", "visit:utm_medium", ["organic"]], - ["is", "visit:utm_term", ["one"]] - ], - "include" => %{"imports" => true} - }) - - assert json_response(conn, 200)["results"] == [ - %{"dimensions" => ["Google"], "metrics" => [1]} - ] - end - - test "ignores imported data if filtered property belongs to a different table than the breakdown property", - %{ - conn: conn, - site: site - } do - site_import = insert(:site_import, site: site) - - populate_stats(site, site_import.id, [ - build(:imported_sources, source: "Google"), - build(:imported_devices, device: "Desktop") - ]) - - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "day", - "dimensions" => ["visit:source"], - "filters" => [ - ["is", "visit:device", ["Desktop"]] - ], - "include" => %{"imports" => true} - }) - - assert %{ - "results" => [], - "meta" => meta - } = json_response(conn, 200) - - assert meta["warning"] =~ "Imported stats are not included in the results" - end - - test "imported country, region and city data", - %{ - conn: conn, - site: site - } do - site_import = insert(:site_import, site: site) - - populate_stats(site, site_import.id, [ - build(:pageview, - timestamp: ~N[2021-01-01 00:15:00], - country_code: "DE", - subdivision1_code: "DE-BE", - city_geoname_id: 2_950_159 - ), - build(:pageview, - timestamp: ~N[2021-01-01 00:15:00], - country_code: "DE", - subdivision1_code: "DE-BE", - city_geoname_id: 2_950_159 - ), - build(:pageview, - timestamp: ~N[2021-01-01 00:15:00], - country_code: "EE", - subdivision1_code: "EE-37", - city_geoname_id: 588_409 - ), - build(:imported_locations, country: "EE", region: "EE-37", city: 588_409, visitors: 33) - ]) - - for {dimension, stats_value, imports_value} <- [ - {"visit:country", "DE", "EE"}, - {"visit:region", "DE-BE", "EE-37"}, - {"visit:city", 2_950_159, 588_409} - ] do - conn = - post(conn, "/api/v2/query", %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "dimensions" => [dimension], - "include" => %{"imports" => true} - }) - - assert json_response(conn, 200)["results"] == [ - %{"dimensions" => [imports_value], "metrics" => [34]}, - %{"dimensions" => [stats_value], "metrics" => [2]} - ] - end - end + assert results == [ + %{"dimensions" => ["Safari"], "metrics" => [2]}, + %{"dimensions" => ["Chrome"], "metrics" => [1]} + ] end test "multiple breakdown timeseries with sources", %{conn: conn, site: site} do diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_validations_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_validations_test.exs new file mode 100644 index 000000000..27c745950 --- /dev/null +++ b/test/plausible_web/controllers/api/external_stats_controller/query_validations_test.exs @@ -0,0 +1,234 @@ +defmodule PlausibleWeb.Api.ExternalStatsController.QueryValidationsTest do + use PlausibleWeb.ConnCase + alias Plausible.Billing.Feature + + setup [:create_user, :create_new_site, :create_api_key, :use_api_key] + + describe "feature access" do + test "cannot break down by a custom prop without access to the props feature", %{ + conn: conn, + user: user, + site: site + } do + ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id) + insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:author"] + }) + + assert json_response(conn, 400)["error"] == + "The owner of this site does not have access to the custom properties feature" + end + + test "can break down by an internal prop key without access to the props feature", %{ + conn: conn, + user: user, + site: site + } do + ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id) + insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:path"] + }) + + assert json_response(conn, 200)["results"] + end + + test "cannot filter by a custom prop without access to the props feature", %{ + conn: conn, + user: user, + site: site + } do + ep = + insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id) + + insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:source"], + "filters" => [["is", "event:props:author", ["Uku"]]] + }) + + assert json_response(conn, 400)["error"] == + "The owner of this site does not have access to the custom properties feature" + end + + test "can filter by an internal prop key without access to the props feature", %{ + conn: conn, + user: user, + site: site + } do + ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id) + insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:source"], + "filters" => [["is", "event:props:url", ["whatever"]]] + }) + + assert json_response(conn, 200)["results"] + end + end + + describe "param validation" do + test "does not allow querying conversion_rate without a goal filter", %{ + conn: conn, + site: site + } do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["conversion_rate"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [["is", "event:props:author", ["Uku"]]] + }) + + assert json_response(conn, 400)["error"] == + "Metric `conversion_rate` can only be queried with event:goal filters or dimensions" + end + + test "validates that dimensions are valid", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["badproperty"] + }) + + assert json_response(conn, 400)["error"] =~ "Invalid dimensions" + end + + test "empty custom property is invalid", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:"] + }) + + assert json_response(conn, 400)["error"] =~ "Invalid dimensions" + end + + test "validates that correct date range is used", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "bad_period", + "dimensions" => ["event:name"] + }) + + assert json_response(conn, 400)["error"] =~ "Invalid date_range" + end + + test "fails when an invalid metric is provided", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "baa"], + "date_range" => "all", + "dimensions" => ["event:name"] + }) + + assert json_response(conn, 400)["error"] =~ "Unknown metric '\"baa\"'" + end + + test "session metrics cannot be used with event:name dimension", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "bounce_rate"], + "date_range" => "all", + "dimensions" => ["event:name"] + }) + + assert json_response(conn, 400)["error"] =~ + "Session metric(s) `bounce_rate` cannot be queried along with event dimensions" + end + + test "session metrics cannot be used with event:props:* dimension", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "bounce_rate"], + "date_range" => "all", + "dimensions" => ["event:props:url"] + }) + + assert json_response(conn, 400)["error"] =~ + "Session metric(s) `bounce_rate` cannot be queried along with event dimensions" + end + + test "validates that metric views_per_visit cannot be used with event:page filter", %{ + conn: conn, + site: site + } do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["views_per_visit"], + "filters" => [["is", "event:page", ["/something"]]] + }) + + assert json_response(conn, 400) == %{ + "error" => + "Metric `views_per_visit` cannot be queried with a filter on `event:page`" + } + end + + test "validates that metric views_per_visit cannot be used together with dimensions", %{ + conn: conn, + site: site + } do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["views_per_visit"], + "dimensions" => ["event:name"] + }) + + assert json_response(conn, 400) == %{ + "error" => "Metric `views_per_visit` cannot be queried with `dimensions`" + } + end + + test "validates a metric can't be asked multiple times", %{ + conn: conn, + site: site + } do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["views_per_visit", "visitors", "visitors"] + }) + + assert json_response(conn, 400) == %{ + "error" => "Metrics cannot be queried multiple times" + } + end + end +end