From 58a66a952c1c2e20358a105d8adf73ee3ba1a7b4 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Tue, 25 Jun 2024 09:27:19 +0300 Subject: [PATCH] APIv2 - initial PR (#4216) * WIP new querying * WIP: Move some aggregate code under new command * WIP: Add joins, handling less metrics * join events table to sessions if needed * Merge imported results with built query * Remove dead code * WIP: /api/v2/query * Allow grouping by time * Use JOIN for main query * Build query result * update parse_time * Make joinless order by work * First test * more breakdown tests * Serialize event:goal filters in an json-encodable way/reflection * Handle inner vs outer ORDER BY clauses properly * Handle single conversion_rate metric * Update more tests * Get parsing tests passing again * Validate filtered goal filter is configured * Enable more validation tests * Enable more event:name breakdown tests * Enable more breakdown tests * Validate site has access to custom props * Validate conversion_rate metric which is only allowed in some situations * Validate that empty event:props: is not valid * handle query.dimensions properly in table_decider * test more validations on metrics/dimensions * Validate session metrics in combination with event dimension(s) * Tests cleanup * Parse include.imports * Get imports working with new querying * Make more imports tests work * Make event:props:path imports-adjacent test work * Get query imports warning-related tests running * Remove dead pagination tests * Solve dead import * Solve some warnings * Update aggregate metrics tests * credo * Improve test naming * Lazy goal loading * Use datetime methods * Ecto -> SQL module name * Remove Expression.dimension mode option --- extra/lib/plausible/stats/goal/revenue.ex | 10 +- lib/plausible/stats.ex | 15 +- lib/plausible/stats/aggregate.ex | 41 +- lib/plausible/stats/base.ex | 95 +- lib/plausible/stats/breakdown.ex | 4 +- lib/plausible/stats/filters/filters.ex | 2 +- lib/plausible/stats/filters/query_parser.ex | 206 +- lib/plausible/stats/imported/imported.ex | 72 +- lib/plausible/stats/metrics.ex | 2 + lib/plausible/stats/query.ex | 19 +- lib/plausible/stats/query_optimizer.ex | 28 + lib/plausible/stats/query_result.ex | 52 + lib/plausible/stats/sql/expression.ex | 89 + lib/plausible/stats/sql/query_builder.ex | 200 ++ lib/plausible/stats/table_decider.ex | 44 +- .../api/external_query_api_controller.ex | 23 + lib/plausible_web/router.ex | 6 + test/plausible/stats/query_parser_test.exs | 417 ++- test/plausible/stats/table_decider_test.exs | 72 +- .../external_stats_controller/query_test.exs | 3106 +++++++++++++++++ 20 files changed, 4236 insertions(+), 267 deletions(-) create mode 100644 lib/plausible/stats/query_optimizer.ex create mode 100644 lib/plausible/stats/query_result.ex create mode 100644 lib/plausible/stats/sql/expression.ex create mode 100644 lib/plausible/stats/sql/query_builder.ex create mode 100644 lib/plausible_web/controllers/api/external_query_api_controller.ex create mode 100644 test/plausible_web/controllers/api/external_stats_controller/query_test.exs diff --git a/extra/lib/plausible/stats/goal/revenue.ex b/extra/lib/plausible/stats/goal/revenue.ex index d531b5d63..7443a1758 100644 --- a/extra/lib/plausible/stats/goal/revenue.ex +++ b/extra/lib/plausible/stats/goal/revenue.ex @@ -15,14 +15,20 @@ defmodule Plausible.Stats.Goal.Revenue do def total_revenue_query() do dynamic( [e], - fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount) + selected_as( + fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount), + :total_revenue + ) ) end def average_revenue_query() do dynamic( [e], - fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount) + selected_as( + fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount), + :average_revenue + ) ) end diff --git a/lib/plausible/stats.ex b/lib/plausible/stats.ex index 42ee7b47c..ffa59389e 100644 --- a/lib/plausible/stats.ex +++ b/lib/plausible/stats.ex @@ -1,12 +1,16 @@ defmodule Plausible.Stats do use Plausible + alias Plausible.Stats.QueryResult + use Plausible.ClickhouseRepo alias Plausible.Stats.{ Breakdown, Aggregate, Timeseries, CurrentVisitors, - FilterSuggestions + FilterSuggestions, + QueryOptimizer, + SQL } use Plausible.DebugReplayInfo @@ -31,6 +35,15 @@ defmodule Plausible.Stats do CurrentVisitors.current_visitors(site) end + def query(site, query) do + optimized_query = QueryOptimizer.optimize(query) + + optimized_query + |> SQL.QueryBuilder.build(site) + |> ClickhouseRepo.all() + |> QueryResult.from(optimized_query) + end + on_ee do def funnel(site, query, funnel) do include_sentry_replay_info() diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex index 01eef54e6..82d1c8a18 100644 --- a/lib/plausible/stats/aggregate.ex +++ b/lib/plausible/stats/aggregate.ex @@ -1,7 +1,7 @@ defmodule Plausible.Stats.Aggregate do use Plausible.ClickhouseRepo use Plausible - import Plausible.Stats.{Base, Imported} + import Plausible.Stats.Base import Ecto.Query alias Plausible.Stats.{Query, Util} @@ -15,23 +15,24 @@ defmodule Plausible.Stats.Aggregate do Query.trace(query, metrics) - {event_metrics, session_metrics, other_metrics} = - metrics - |> Util.maybe_add_visitors_metric() - |> Plausible.Stats.TableDecider.partition_metrics(query) + query_with_metrics = %Plausible.Stats.Query{ + query + | metrics: Util.maybe_add_visitors_metric(metrics) + } - event_task = fn -> aggregate_events(site, query, event_metrics) end - - session_task = fn -> aggregate_sessions(site, query, session_metrics) end + q = Plausible.Stats.SQL.QueryBuilder.build(query_with_metrics, site) time_on_page_task = - if :time_on_page in other_metrics do + if :time_on_page in query_with_metrics.metrics do fn -> aggregate_time_on_page(site, query) end else fn -> %{} end end - Plausible.ClickhouseRepo.parallel_tasks([session_task, event_task, time_on_page_task]) + Plausible.ClickhouseRepo.parallel_tasks([ + run_query_task(q), + time_on_page_task + ]) |> Enum.reduce(%{}, fn aggregate, task_result -> Map.merge(aggregate, task_result) end) |> Util.keep_requested_metrics(metrics) |> cast_revenue_metrics_to_money(currency) @@ -40,24 +41,8 @@ defmodule Plausible.Stats.Aggregate do |> Enum.into(%{}) end - defp aggregate_events(_, _, []), do: %{} - - defp aggregate_events(site, query, metrics) do - from(e in base_event_query(site, query), select: ^select_event_metrics(metrics)) - |> merge_imported(site, query, metrics) - |> maybe_add_conversion_rate(site, query, metrics) - |> ClickhouseRepo.one() - end - - defp aggregate_sessions(_, _, []), do: %{} - - defp aggregate_sessions(site, query, metrics) do - from(e in query_sessions(site, query), select: ^select_session_metrics(metrics, query)) - |> filter_converted_sessions(site, query) - |> merge_imported(site, query, metrics) - |> ClickhouseRepo.one() - |> Util.keep_requested_metrics(metrics) - end + defp run_query_task(nil), do: fn -> %{} end + defp run_query_task(q), do: fn -> ClickhouseRepo.one(q) end defp aggregate_time_on_page(site, query) do windowed_pages_q = diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index a449d0fd3..bd01257fb 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -31,7 +31,7 @@ defmodule Plausible.Stats.Base do end end - defp query_events(site, query) do + def query_events(site, query) do q = from(e in "events_v2", where: ^Filters.WhereBuilder.build(:events, site, query)) on_ee do @@ -62,14 +62,21 @@ defmodule Plausible.Stats.Base do pageviews: dynamic( [e], - fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name) + selected_as( + fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name), + :pageviews + ) ) } end defp select_event_metric(:events) do %{ - events: dynamic([], fragment("toUInt64(round(count(*) * any(_sample_factor)))")) + events: + dynamic( + [], + selected_as(fragment("toUInt64(round(count(*) * any(_sample_factor)))"), :events) + ) } end @@ -82,7 +89,13 @@ defmodule Plausible.Stats.Base do defp select_event_metric(:visits) do %{ visits: - dynamic([e], fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.session_id)) + dynamic( + [e], + selected_as( + fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.session_id), + :visits + ) + ) } end @@ -127,10 +140,13 @@ defmodule Plausible.Stats.Base do bounce_rate: dynamic( [], - fragment( - "toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))", - ^condition, - ^condition + selected_as( + fragment( + "toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))", + ^condition, + ^condition + ), + :bounce_rate ) ), __internal_visits: dynamic([], fragment("toUInt32(sum(sign))")) @@ -139,7 +155,14 @@ defmodule Plausible.Stats.Base do defp select_session_metric(:visits, _query) do %{ - visits: dynamic([s], fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign)) + visits: + dynamic( + [s], + selected_as( + fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign), + :visits + ) + ) } end @@ -148,7 +171,10 @@ defmodule Plausible.Stats.Base do pageviews: dynamic( [s], - fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews) + selected_as( + fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews), + :pageviews + ) ) } end @@ -158,7 +184,10 @@ defmodule Plausible.Stats.Base do events: dynamic( [s], - fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events) + selected_as( + fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events), + :events + ) ) } end @@ -179,7 +208,13 @@ defmodule Plausible.Stats.Base do defp select_session_metric(:visit_duration, _query) do %{ visit_duration: - dynamic([], fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))")), + dynamic( + [], + selected_as( + fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))"), + :visit_duration + ) + ), __internal_visits: dynamic([], fragment("toUInt32(sum(sign))")) } end @@ -189,7 +224,15 @@ defmodule Plausible.Stats.Base do views_per_visit: dynamic( [s], - fragment("ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)", s.sign, s.pageviews, s.sign) + selected_as( + fragment( + "ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)", + s.sign, + s.pageviews, + s.sign + ), + :views_per_visit + ) ), __internal_visits: dynamic([], fragment("toUInt32(sum(sign))")) } @@ -328,11 +371,14 @@ defmodule Plausible.Stats.Base do ) |> select_merge(%{ percentage: - fragment( - "if(? > 0, round(? / ? * 100, 1), null)", - selected_as(:__total_visitors), - selected_as(:visitors), - selected_as(:__total_visitors) + selected_as( + fragment( + "if(? > 0, round(? / ? * 100, 1), null)", + selected_as(:__total_visitors), + selected_as(:visitors), + selected_as(:__total_visitors) + ), + :percentage ) }) else @@ -357,11 +403,14 @@ defmodule Plausible.Stats.Base do ) |> select_merge([e], %{ conversion_rate: - fragment( - "if(? > 0, round(? / ? * 100, 1), 0)", - selected_as(:__total_visitors), - e.visitors, - selected_as(:__total_visitors) + selected_as( + fragment( + "if(? > 0, round(? / ? * 100, 1), 0)", + selected_as(:__total_visitors), + e.visitors, + selected_as(:__total_visitors) + ), + :conversion_rate ) }) else diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 6f3799e83..d68e9fcad 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -276,8 +276,8 @@ defmodule Plausible.Stats.Breakdown do defp breakdown_table(%Query{dimensions: ["visit:exit_page"]}, _metrics), do: :session defp breakdown_table(%Query{dimensions: ["visit:exit_page_hostname"]}, _metrics), do: :session - defp breakdown_table(%Query{dimensions: [dimension]} = query, metrics) do - {_, session_metrics, _} = TableDecider.partition_metrics(metrics, query, dimension) + defp breakdown_table(%Query{dimensions: [_dimension]} = query, metrics) do + {_, session_metrics, _} = TableDecider.partition_metrics(metrics, query) if not Enum.empty?(session_metrics) do :session diff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.ex index b6760965e..dc13c7dcd 100644 --- a/lib/plausible/stats/filters/filters.ex +++ b/lib/plausible/stats/filters/filters.ex @@ -87,6 +87,6 @@ defmodule Plausible.Stats.Filters do property |> String.split(":") |> List.last() - |> String.to_existing_atom() + |> String.to_atom() end end diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.ex index b2c7690e5..502326189 100644 --- a/lib/plausible/stats/filters/query_parser.ex +++ b/lib/plausible/stats/filters/query_parser.ex @@ -1,22 +1,30 @@ defmodule Plausible.Stats.Filters.QueryParser do @moduledoc false + alias Plausible.Stats.TableDecider alias Plausible.Stats.Filters + alias Plausible.Stats.Query - def parse(params) when is_map(params) do + def parse(site, params) when is_map(params) do with {:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])), {:ok, filters} <- parse_filters(Map.get(params, "filters", [])), - {:ok, date_range} <- parse_date_range(Map.get(params, "date_range")), + {:ok, date_range} <- parse_date_range(site, Map.get(params, "date_range")), {:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])), {:ok, order_by} <- parse_order_by(Map.get(params, "order_by")), + {:ok, include} <- parse_include(Map.get(params, "include", %{})), query = %{ metrics: metrics, filters: filters, date_range: date_range, dimensions: dimensions, - order_by: order_by + order_by: order_by, + timezone: site.timezone, + imported_data_requested: Map.get(include, :imports, false) }, - :ok <- validate_order_by(query) do + :ok <- validate_order_by(query), + :ok <- validate_goal_filters(site, query), + :ok <- validate_custom_props_access(site, query), + :ok <- validate_metrics(query) do {:ok, query} end end @@ -87,16 +95,26 @@ defmodule Plausible.Stats.Filters.QueryParser do defp parse_clauses_list(filter), do: {:error, "Invalid filter '#{inspect(filter)}'"} - defp parse_date_range("day"), do: {:ok, "day"} - defp parse_date_range("7d"), do: {:ok, "7d"} - defp parse_date_range("30d"), do: {:ok, "30d"} - defp parse_date_range("month"), do: {:ok, "month"} - defp parse_date_range("6mo"), do: {:ok, "6mo"} - defp parse_date_range("12mo"), do: {:ok, "6mo"} - defp parse_date_range("year"), do: {:ok, "year"} - defp parse_date_range("all"), do: {:ok, "all"} + defp parse_date_range(site, "day") do + today = DateTime.now!(site.timezone) |> DateTime.to_date() + {:ok, Date.range(today, today)} + end - defp parse_date_range([from_date_string, to_date_string]) + defp parse_date_range(_site, "7d"), do: {:ok, "7d"} + defp parse_date_range(_site, "30d"), do: {:ok, "30d"} + defp parse_date_range(_site, "month"), do: {:ok, "month"} + defp parse_date_range(_site, "6mo"), do: {:ok, "6mo"} + defp parse_date_range(_site, "12mo"), do: {:ok, "12mo"} + defp parse_date_range(_site, "year"), do: {:ok, "year"} + + defp parse_date_range(site, "all") do + today = DateTime.now!(site.timezone) |> DateTime.to_date() + start_date = Plausible.Sites.stats_start_date(site) || today + + {:ok, Date.range(start_date, today)} + end + + defp parse_date_range(_site, [from_date_string, to_date_string]) when is_bitstring(from_date_string) and is_bitstring(to_date_string) do with {:ok, from_date} <- Date.from_iso8601(from_date_string), {:ok, to_date} <- Date.from_iso8601(to_date_string) do @@ -106,13 +124,13 @@ defmodule Plausible.Stats.Filters.QueryParser do end end - defp parse_date_range(unknown), do: {:error, "Invalid date range '#{inspect(unknown)}'"} + defp parse_date_range(_site, unknown), do: {:error, "Invalid date range '#{inspect(unknown)}'"} defp parse_dimensions(dimensions) when is_list(dimensions) do if length(dimensions) == length(Enum.uniq(dimensions)) do parse_list( dimensions, - &parse_filter_key_string(&1, "Invalid dimensions '#{inspect(dimensions)}'") + &parse_dimension_entry(&1, "Invalid dimensions '#{inspect(dimensions)}'") ) else {:error, "Some dimensions are listed multiple times"} @@ -121,36 +139,67 @@ defmodule Plausible.Stats.Filters.QueryParser do defp parse_dimensions(dimensions), do: {:error, "Invalid dimensions '#{inspect(dimensions)}'"} - def parse_order_by(order_by) when is_list(order_by) do + defp parse_order_by(order_by) when is_list(order_by) do parse_list(order_by, &parse_order_by_entry/1) end - def parse_order_by(nil), do: {:ok, nil} - def parse_order_by(order_by), do: {:error, "Invalid order_by '#{inspect(order_by)}'"} + defp parse_order_by(nil), do: {:ok, nil} + defp parse_order_by(order_by), do: {:error, "Invalid order_by '#{inspect(order_by)}'"} - def parse_order_by_entry(entry) do - with {:ok, metric_or_dimension} <- parse_metric_or_dimension(entry), + defp parse_order_by_entry(entry) do + with {:ok, value} <- parse_metric_or_dimension(entry), {:ok, order_direction} <- parse_order_direction(entry) do - {:ok, {metric_or_dimension, order_direction}} + {:ok, {value, order_direction}} end end - def parse_metric_or_dimension([metric_or_dimension, _] = entry) do - case {parse_metric(metric_or_dimension), parse_filter_key_string(metric_or_dimension)} do - {{:ok, metric}, _} -> {:ok, metric} - {_, {:ok, dimension}} -> {:ok, dimension} + defp parse_dimension_entry(key, error_message) do + case { + parse_time(key), + parse_filter_key_string(key) + } do + {{:ok, time}, _} -> {:ok, time} + {_, {:ok, filter_key}} -> {:ok, filter_key} + _ -> {:error, error_message} + end + end + + defp parse_metric_or_dimension([value, _] = entry) do + case { + parse_time(value), + parse_metric(value), + parse_filter_key_string(value) + } do + {{:ok, time}, _, _} -> {:ok, time} + {_, {:ok, metric}, _} -> {:ok, metric} + {_, _, {:ok, dimension}} -> {:ok, dimension} _ -> {:error, "Invalid order_by entry '#{inspect(entry)}'"} end end - def parse_order_direction([_, "asc"]), do: {:ok, :asc} - def parse_order_direction([_, "desc"]), do: {:ok, :desc} - def parse_order_direction(entry), do: {:error, "Invalid order_by entry '#{inspect(entry)}'"} + defp parse_time("time"), do: {:ok, "time"} + defp parse_time("time:day"), do: {:ok, "time:day"} + defp parse_time("time:week"), do: {:ok, "time:week"} + defp parse_time("time:month"), do: {:ok, "time:month"} + defp parse_time("time:year"), do: {:ok, "time:year"} + defp parse_time(_), do: :error + + defp parse_order_direction([_, "asc"]), do: {:ok, :asc} + defp parse_order_direction([_, "desc"]), do: {:ok, :desc} + defp parse_order_direction(entry), do: {:error, "Invalid order_by entry '#{inspect(entry)}'"} + + defp parse_include(%{"imports" => value}) when is_boolean(value), do: {:ok, %{imports: value}} + defp parse_include(%{}), do: {:ok, %{}} + defp parse_include(include), do: {:error, "Invalid include passed '#{inspect(include)}'"} defp parse_filter_key_string(filter_key, error_message \\ "") do case filter_key do - "event:props:" <> _property_name -> - {:ok, filter_key} + "event:props:" <> property_name -> + if String.length(property_name) > 0 do + {:ok, filter_key} + else + {:error, error_message} + end "event:" <> key -> if key in Filters.event_props() do @@ -171,7 +220,7 @@ defmodule Plausible.Stats.Filters.QueryParser do end end - def validate_order_by(query) do + defp validate_order_by(query) do if query.order_by do valid_values = query.metrics ++ query.dimensions @@ -193,6 +242,92 @@ defmodule Plausible.Stats.Filters.QueryParser do end end + defp validate_goal_filters(site, query) do + goal_filter_clauses = + Enum.flat_map(query.filters, fn + [_operation, "event:goal", clauses] -> clauses + _ -> [] + end) + + if length(goal_filter_clauses) > 0 do + configured_goals = + Plausible.Goals.for_site(site) + |> Enum.map(fn + %{page_path: path} when is_binary(path) -> {:page, path} + %{event_name: event_name} -> {:event, event_name} + end) + + validate_list(goal_filter_clauses, &validate_goal_filter(&1, configured_goals)) + else + :ok + end + end + + defp validate_goal_filter(clause, configured_goals) do + if Enum.member?(configured_goals, clause) do + :ok + else + {:error, + "The goal `#{Filters.Utils.unwrap_goal_value(clause)}` is not configured for this site. Find out how to configure goals here: https://plausible.io/docs/stats-api#filtering-by-goals"} + end + end + + defp validate_custom_props_access(site, query) do + allowed_props = Plausible.Props.allowed_for(site, bypass_setup?: true) + + validate_custom_props_access(site, query, allowed_props) + end + + defp validate_custom_props_access(_site, _query, :all), do: :ok + + defp validate_custom_props_access(_site, query, allowed_props) do + valid? = + query.filters + |> Enum.map(fn [_operation, filter_key | _rest] -> filter_key end) + |> Enum.concat(query.dimensions) + |> Enum.all?(fn + "event:props:" <> prop -> prop in allowed_props + _ -> true + end) + + if valid? do + :ok + else + {:error, "The owner of this site does not have access to the custom properties feature"} + end + end + + defp validate_metrics(query) do + validate_list(query.metrics, &validate_metric(&1, query)) + + with :ok <- validate_list(query.metrics, &validate_metric(&1, query)) do + validate_no_metrics_filters_conflict(query) + end + end + + defp validate_metric(:conversion_rate = metric, query) do + if Enum.member?(query.dimensions, "event:goal") or + not is_nil(Query.get_filter(query, "event:goal")) do + :ok + else + {:error, "Metric `#{metric}` can only be queried with event:goal filters or dimensions"} + end + end + + defp validate_metric(_, _), do: :ok + + defp validate_no_metrics_filters_conflict(query) do + {_event_metrics, sessions_metrics, _other_metrics} = + TableDecider.partition_metrics(query.metrics, query) + + if Enum.empty?(sessions_metrics) or not TableDecider.event_filters?(query) do + :ok + else + {:error, + "Session metric(s) `#{sessions_metrics |> Enum.join(", ")}` cannot be queried along with event filters or dimensions"} + end + end + defp parse_list(list, parser_function) do Enum.reduce_while(list, {:ok, []}, fn value, {:ok, results} -> case parser_function.(value) do @@ -201,4 +336,13 @@ defmodule Plausible.Stats.Filters.QueryParser do end end) end + + defp validate_list(list, parser_function) do + Enum.reduce_while(list, :ok, fn value, :ok -> + case parser_function.(value) do + :ok -> {:cont, :ok} + {:error, _} = error -> {:halt, error} + end + end) + end end diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.ex index 2e680f824..5c1d77cc3 100644 --- a/lib/plausible/stats/imported/imported.ex +++ b/lib/plausible/stats/imported/imported.ex @@ -274,18 +274,18 @@ defmodule Plausible.Stats.Imported do site |> Imported.Base.query_imported(query) |> where([i], i.visitors > 0) - |> group_imported_by(dim) + |> group_imported_by(dim, query) |> select_imported_metrics(metrics) join_on = case dim do - _ when dim in [:url, :path] -> + _ when dim in [:url, :path] and not query.v2 -> dynamic([s, i], s.breakdown_prop_value == i.breakdown_prop_value) - :os_version -> + :os_version when not query.v2 -> dynamic([s, i], s.os == i.os and s.os_version == i.os_version) - :browser_version -> + :browser_version when not query.v2 -> dynamic([s, i], s.browser == i.browser and s.browser_version == i.browser_version) dim -> @@ -297,7 +297,7 @@ defmodule Plausible.Stats.Imported do on: ^join_on, select: %{} ) - |> select_joined_dimension(dim) + |> select_joined_dimension(dim, query) |> select_joined_metrics(metrics) |> apply_order_by(metrics) end @@ -344,7 +344,7 @@ defmodule Plausible.Stats.Imported do on: s.name == i.name, select: %{} ) - |> select_joined_dimension(:name) + |> select_joined_dimension(:name, query) |> select_joined_metrics(metrics) else q @@ -551,7 +551,7 @@ defmodule Plausible.Stats.Imported do |> select_imported_metrics(rest) end - defp group_imported_by(q, dim) when dim in [:source, :referrer] do + defp group_imported_by(q, dim, _query) when dim in [:source, :referrer] do q |> group_by([i], field(i, ^dim)) |> select_merge([i], %{ @@ -559,7 +559,7 @@ defmodule Plausible.Stats.Imported do }) end - defp group_imported_by(q, dim) + defp group_imported_by(q, dim, _query) when dim in [:utm_source, :utm_medium, :utm_campaign, :utm_term, :utm_content] do q |> group_by([i], field(i, ^dim)) @@ -567,34 +567,34 @@ defmodule Plausible.Stats.Imported do |> select_merge([i], %{^dim => field(i, ^dim)}) end - defp group_imported_by(q, :page) do + defp group_imported_by(q, :page, _query) do q |> group_by([i], i.page) |> select_merge([i], %{page: i.page, time_on_page: sum(i.time_on_page)}) end - defp group_imported_by(q, :country) do + defp group_imported_by(q, :country, _query) do q |> group_by([i], i.country) |> where([i], i.country != "ZZ") |> select_merge([i], %{country: i.country}) end - defp group_imported_by(q, :region) do + defp group_imported_by(q, :region, _query) do q |> group_by([i], i.region) |> where([i], i.region != "") |> select_merge([i], %{region: i.region}) end - defp group_imported_by(q, :city) do + defp group_imported_by(q, :city, _query) do q |> group_by([i], i.city) |> where([i], i.city != 0 and not is_nil(i.city)) |> select_merge([i], %{city: i.city}) end - defp group_imported_by(q, dim) when dim in [:device, :browser] do + defp group_imported_by(q, dim, _query) when dim in [:device, :browser] do q |> group_by([i], field(i, ^dim)) |> select_merge([i], %{ @@ -602,7 +602,7 @@ defmodule Plausible.Stats.Imported do }) end - defp group_imported_by(q, :browser_version) do + defp group_imported_by(q, :browser_version, _query) do q |> group_by([i], [i.browser, i.browser_version]) |> select_merge([i], %{ @@ -617,7 +617,7 @@ defmodule Plausible.Stats.Imported do }) end - defp group_imported_by(q, :os) do + defp group_imported_by(q, :os, _query) do q |> group_by([i], i.operating_system) |> select_merge([i], %{ @@ -625,7 +625,7 @@ defmodule Plausible.Stats.Imported do }) end - defp group_imported_by(q, :os_version) do + defp group_imported_by(q, :os_version, _query) do q |> group_by([i], [i.operating_system, i.operating_system_version]) |> select_merge([i], %{ @@ -640,19 +640,27 @@ defmodule Plausible.Stats.Imported do }) end - defp group_imported_by(q, dim) when dim in [:entry_page, :exit_page] do + defp group_imported_by(q, dim, _query) when dim in [:entry_page, :exit_page] do q |> group_by([i], field(i, ^dim)) |> select_merge([i], %{^dim => field(i, ^dim)}) end - defp group_imported_by(q, :name) do + defp group_imported_by(q, :name, _query) do q |> group_by([i], i.name) |> select_merge([i], %{name: i.name}) end - defp group_imported_by(q, :url) do + defp group_imported_by(q, :url, query) when query.v2 do + q + |> group_by([i], i.link_url) + |> select_merge([i], %{ + url: fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none) + }) + end + + defp group_imported_by(q, :url, _query) do q |> group_by([i], i.link_url) |> select_merge([i], %{ @@ -660,7 +668,15 @@ defmodule Plausible.Stats.Imported do }) end - defp group_imported_by(q, :path) do + defp group_imported_by(q, :path, query) when query.v2 do + q + |> group_by([i], i.path) + |> select_merge([i], %{ + path: fragment("if(not empty(?), ?, ?)", i.path, i.path, @none) + }) + end + + defp group_imported_by(q, :path, _query) do q |> group_by([i], i.path) |> select_merge([i], %{ @@ -668,20 +684,20 @@ defmodule Plausible.Stats.Imported do }) end - defp select_joined_dimension(q, :city) do + defp select_joined_dimension(q, :city, _query) do select_merge(q, [s, i], %{ city: fragment("greatest(?,?)", i.city, s.city) }) end - defp select_joined_dimension(q, :os_version) do + defp select_joined_dimension(q, :os_version, query) when not query.v2 do select_merge(q, [s, i], %{ os: fragment("if(empty(?), ?, ?)", s.os, i.os, s.os), os_version: fragment("if(empty(?), ?, ?)", s.os_version, i.os_version, s.os_version) }) end - defp select_joined_dimension(q, :browser_version) do + defp select_joined_dimension(q, :browser_version, query) when not query.v2 do select_merge(q, [s, i], %{ browser: fragment("if(empty(?), ?, ?)", s.browser, i.browser, s.browser), browser_version: @@ -689,7 +705,7 @@ defmodule Plausible.Stats.Imported do }) end - defp select_joined_dimension(q, dim) when dim in [:url, :path] do + defp select_joined_dimension(q, dim, query) when dim in [:url, :path] and not query.v2 do select_merge(q, [s, i], %{ breakdown_prop_value: fragment( @@ -701,7 +717,7 @@ defmodule Plausible.Stats.Imported do }) end - defp select_joined_dimension(q, dim) do + defp select_joined_dimension(q, dim, _query) do select_merge(q, [s, i], %{ ^dim => fragment("if(empty(?), ?, ?)", field(s, ^dim), field(i, ^dim), field(s, ^dim)) }) @@ -716,7 +732,7 @@ defmodule Plausible.Stats.Imported do defp select_joined_metrics(q, [:visits | rest]) do q - |> select_merge([s, i], %{visits: s.visits + i.visits}) + |> select_merge([s, i], %{visits: selected_as(s.visits + i.visits, :visits)}) |> select_joined_metrics(rest) end @@ -728,13 +744,13 @@ defmodule Plausible.Stats.Imported do defp select_joined_metrics(q, [:events | rest]) do q - |> select_merge([s, i], %{events: s.events + i.events}) + |> select_merge([s, i], %{events: selected_as(s.events + i.events, :events)}) |> select_joined_metrics(rest) end defp select_joined_metrics(q, [:pageviews | rest]) do q - |> select_merge([s, i], %{pageviews: s.pageviews + i.pageviews}) + |> select_merge([s, i], %{pageviews: selected_as(s.pageviews + i.pageviews, :pageviews)}) |> select_joined_metrics(rest) end diff --git a/lib/plausible/stats/metrics.ex b/lib/plausible/stats/metrics.ex index 36a1c3743..733c7d6e1 100644 --- a/lib/plausible/stats/metrics.ex +++ b/lib/plausible/stats/metrics.ex @@ -21,6 +21,8 @@ defmodule Plausible.Stats.Metrics do @metric_mappings Enum.into(@all_metrics, %{}, fn metric -> {to_string(metric), metric} end) + def metric?(value), do: Enum.member?(@all_metrics, value) + def from_string!(str) do Map.fetch!(@metric_mappings, str) end diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index 737b3ba0b..ae25c6536 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -13,7 +13,11 @@ defmodule Plausible.Stats.Query do now: nil, experimental_session_count?: false, experimental_reduced_joins?: false, - latest_import_end_date: nil + latest_import_end_date: nil, + metrics: [], + order_by: [], + timezone: nil, + v2: false require OpenTelemetry.Tracer, as: Tracer alias Plausible.Stats.{Filters, Interval, Imported} @@ -42,6 +46,19 @@ defmodule Plausible.Stats.Query do query end + def build(site, params) do + with {:ok, query_data} <- Filters.QueryParser.parse(site, params) do + query = + struct!(__MODULE__, Map.to_list(query_data)) + |> put_imported_opts(site, %{}) + |> put_experimental_session_count(site, params) + |> put_experimental_reduced_joins(site, params) + |> struct!(v2: true) + + {:ok, query} + end + end + defp put_experimental_session_count(query, site, params) do if Map.has_key?(params, "experimental_session_count") do struct!(query, diff --git a/lib/plausible/stats/query_optimizer.ex b/lib/plausible/stats/query_optimizer.ex new file mode 100644 index 000000000..936fa2758 --- /dev/null +++ b/lib/plausible/stats/query_optimizer.ex @@ -0,0 +1,28 @@ +defmodule Plausible.Stats.QueryOptimizer do + @moduledoc false + + alias Plausible.Stats.Query + + def optimize(query) do + Enum.reduce(pipeline(), query, fn step, acc -> step.(acc) end) + end + + defp pipeline() do + [ + &add_missing_order_by/1, + &update_group_by_time/1 + ] + end + + defp add_missing_order_by(%Query{order_by: nil} = query) do + %Query{query | order_by: [{hd(query.metrics), :desc}]} + end + + defp add_missing_order_by(query), do: query + + defp update_group_by_time(%Query{dimensions: ["time" | rest]} = query) do + %Query{query | dimensions: ["time:month" | rest]} + end + + defp update_group_by_time(query), do: query +end diff --git a/lib/plausible/stats/query_result.ex b/lib/plausible/stats/query_result.ex new file mode 100644 index 000000000..d7b72bf80 --- /dev/null +++ b/lib/plausible/stats/query_result.ex @@ -0,0 +1,52 @@ +defmodule Plausible.Stats.QueryResult do + @moduledoc false + + alias Plausible.Stats.SQL.QueryBuilder + alias Plausible.Stats.Filters + alias Plausible.Stats.Query + + @derive Jason.Encoder + defstruct results: [], + query: nil, + meta: %{} + + def from(results, query) do + results_list = + results + |> Enum.map(fn entry -> + %{ + dimensions: Enum.map(query.dimensions, &Map.get(entry, QueryBuilder.shortname(&1))), + metrics: Enum.map(query.metrics, &Map.get(entry, &1)) + } + end) + + struct!( + __MODULE__, + results: results_list, + query: %{ + metrics: query.metrics, + date_range: [query.date_range.first, query.date_range.last], + filters: query.filters |> Enum.map(&serializable_filter/1), + dimensions: query.dimensions, + order_by: query.order_by |> Enum.map(&Tuple.to_list/1) + }, + meta: meta(query) + ) + end + + defp meta(%Query{skip_imported_reason: :unsupported_query}) do + %{ + warning: + "Imported stats are not included in the results because query parameters are not supported. " <> + "For more information, see: https://plausible.io/docs/stats-api#filtering-imported-stats" + } + end + + defp meta(_), do: %{} + + defp serializable_filter([operation, "event:goal", clauses]) do + [operation, "event:goal", Enum.map(clauses, &Filters.Utils.unwrap_goal_value/1)] + end + + defp serializable_filter(filter), do: filter +end diff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.ex new file mode 100644 index 000000000..cdd1cffe4 --- /dev/null +++ b/lib/plausible/stats/sql/expression.ex @@ -0,0 +1,89 @@ +defmodule Plausible.Stats.SQL.Expression do + @moduledoc false + + import Ecto.Query + + use Plausible.Stats.Fragments + + @no_ref "Direct / None" + @not_set "(not set)" + + defmacrop field_or_blank_value(expr, empty_value) do + quote do + dynamic( + [t], + fragment("if(empty(?), ?, ?)", unquote(expr), unquote(empty_value), unquote(expr)) + ) + end + end + + def dimension("time:hour", query) do + dynamic([t], fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone)) + end + + def dimension("time:day", query) do + dynamic([t], fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone)) + end + + def dimension("time:month", query) do + dynamic([t], fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone)) + end + + def dimension("event:name", _query), do: dynamic([t], t.name) + def dimension("event:page", _query), do: dynamic([t], t.pathname) + def dimension("event:hostname", _query), do: dynamic([t], t.hostname) + + def dimension("event:props:" <> property_name, _query) do + dynamic( + [t], + fragment( + "if(not empty(?), ?, '(none)')", + get_by_key(t, :meta, ^property_name), + get_by_key(t, :meta, ^property_name) + ) + ) + end + + def dimension("visit:entry_page", _query), do: dynamic([t], t.entry_page) + def dimension("visit:exit_page", _query), do: dynamic([t], t.exit_page) + + def dimension("visit:utm_medium", _query), + do: field_or_blank_value(t.utm_medium, @not_set) + + def dimension("visit:utm_source", _query), + do: field_or_blank_value(t.utm_source, @not_set) + + def dimension("visit:utm_campaign", _query), + do: field_or_blank_value(t.utm_campaign, @not_set) + + def dimension("visit:utm_content", _query), + do: field_or_blank_value(t.utm_content, @not_set) + + def dimension("visit:utm_term", _query), + do: field_or_blank_value(t.utm_term, @not_set) + + def dimension("visit:source", _query), + do: field_or_blank_value(t.source, @no_ref) + + def dimension("visit:referrer", _query), + do: field_or_blank_value(t.referrer, @no_ref) + + def dimension("visit:device", _query), + do: field_or_blank_value(t.device, @not_set) + + def dimension("visit:os", _query), do: field_or_blank_value(t.os, @not_set) + + def dimension("visit:os_version", _query), + do: field_or_blank_value(t.os_version, @not_set) + + def dimension("visit:browser", _query), + do: field_or_blank_value(t.browser, @not_set) + + def dimension("visit:browser_version", _query), + do: field_or_blank_value(t.browser_version, @not_set) + + # :TODO: Locations also set extra filters + def dimension("visit:country", _query), do: dynamic([t], t.country) + def dimension("visit:region", _query), do: dynamic([t], t.region) + def dimension("visit:city", _query), do: dynamic([t], t.city) +end diff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.ex new file mode 100644 index 000000000..a966be4b1 --- /dev/null +++ b/lib/plausible/stats/sql/query_builder.ex @@ -0,0 +1,200 @@ +defmodule Plausible.Stats.SQL.QueryBuilder do + @moduledoc false + + use Plausible + + import Ecto.Query + import Plausible.Stats.Imported + + alias Plausible.Stats.{Base, Query, TableDecider, Util, Filters, Metrics} + alias Plausible.Stats.SQL.Expression + + def build(query, site) do + {event_metrics, sessions_metrics, _other_metrics} = + query.metrics + |> Util.maybe_add_visitors_metric() + |> TableDecider.partition_metrics(query) + + join_query_results( + build_events_query(site, query, event_metrics), + event_metrics, + build_sessions_query(site, query, sessions_metrics), + sessions_metrics, + query + ) + end + + def shortname(metric) when is_atom(metric), do: metric + def shortname(dimension), do: Plausible.Stats.Filters.without_prefix(dimension) + + defp build_events_query(_, _, []), do: nil + + defp build_events_query(site, query, event_metrics) do + q = + from( + e in "events_v2", + where: ^Filters.WhereBuilder.build(:events, site, query), + select: ^Base.select_event_metrics(event_metrics) + ) + + on_ee do + q = Plausible.Stats.Sampling.add_query_hint(q, query) + end + + q + |> join_sessions_if_needed(site, query) + |> build_group_by(query) + |> merge_imported(site, query, event_metrics) + |> Base.maybe_add_conversion_rate(site, query, event_metrics) + end + + defp join_sessions_if_needed(q, site, query) do + if TableDecider.events_join_sessions?(query) do + sessions_q = + from( + s in Base.query_sessions(site, query), + select: %{session_id: s.session_id}, + where: s.sign == 1, + group_by: s.session_id + ) + + from( + e in q, + join: sq in subquery(sessions_q), + on: e.session_id == sq.session_id + ) + else + q + end + end + + def build_sessions_query(_, _, []), do: nil + + def build_sessions_query(site, query, session_metrics) do + q = + from( + e in "sessions_v2", + where: ^Filters.WhereBuilder.build(:sessions, site, query), + select: ^Base.select_session_metrics(session_metrics, query) + ) + + on_ee do + q = Plausible.Stats.Sampling.add_query_hint(q, query) + end + + q + |> join_events_if_needed(site, query) + |> build_group_by(query) + |> merge_imported(site, query, session_metrics) + end + + def join_events_if_needed(q, site, query) do + if Query.has_event_filters?(query) do + events_q = + from(e in "events_v2", + where: ^Filters.WhereBuilder.build(:events, site, query), + select: %{ + session_id: fragment("DISTINCT ?", e.session_id), + _sample_factor: fragment("_sample_factor") + } + ) + + on_ee do + events_q = Plausible.Stats.Sampling.add_query_hint(events_q, query) + end + + from(s in q, + join: e in subquery(events_q), + on: s.session_id == e.session_id + ) + else + q + end + end + + defp build_group_by(q, query) do + Enum.reduce(query.dimensions, q, fn dimension, q -> + q + |> select_merge(^%{shortname(dimension) => Expression.dimension(dimension, query)}) + |> group_by(^Expression.dimension(dimension, query)) + end) + end + + defp build_order_by(q, query, mode) do + Enum.reduce(query.order_by, q, &build_order_by(&2, query, &1, mode)) + end + + def build_order_by(q, query, {metric_or_dimension, order_direction}, :inner) do + order_by( + q, + [t], + ^{ + order_direction, + if( + Metrics.metric?(metric_or_dimension), + do: dynamic([], selected_as(^shortname(metric_or_dimension))), + else: Expression.dimension(metric_or_dimension, query) + ) + } + ) + end + + def build_order_by(q, _query, {metric_or_dimension, order_direction}, :outer) do + order_by( + q, + [t], + ^{ + order_direction, + dynamic([], selected_as(^shortname(metric_or_dimension))) + } + ) + end + + defmacrop select_join_fields(q, list, table_name) do + quote do + Enum.reduce(unquote(list), unquote(q), fn metric_or_dimension, q -> + select_merge( + q, + ^%{ + shortname(metric_or_dimension) => + dynamic( + [e, s], + selected_as( + field(unquote(table_name), ^shortname(metric_or_dimension)), + ^shortname(metric_or_dimension) + ) + ) + } + ) + end) + end + end + + defp join_query_results(nil, _, nil, _, _query), do: nil + + defp join_query_results(events_q, _, nil, _, query), + do: events_q |> build_order_by(query, :inner) + + defp join_query_results(nil, _, sessions_q, _, query), + do: sessions_q |> build_order_by(query, :inner) + + defp join_query_results(events_q, event_metrics, sessions_q, sessions_metrics, query) do + join(subquery(events_q), :left, [e], s in subquery(sessions_q), + on: ^build_group_by_join(query) + ) + |> select_join_fields(query.dimensions, e) + |> select_join_fields(event_metrics, e) + |> select_join_fields(List.delete(sessions_metrics, :sample_percent), s) + |> build_order_by(query, :outer) + end + + defp build_group_by_join(%Query{dimensions: []}), do: true + + defp build_group_by_join(query) do + query.dimensions + |> Enum.map(fn dim -> + dynamic([e, s], field(e, ^shortname(dim)) == field(s, ^shortname(dim))) + end) + |> Enum.reduce(fn condition, acc -> dynamic([], ^acc and ^condition) end) + end +end diff --git a/lib/plausible/stats/table_decider.ex b/lib/plausible/stats/table_decider.ex index 8b77ca592..629744228 100644 --- a/lib/plausible/stats/table_decider.ex +++ b/lib/plausible/stats/table_decider.ex @@ -9,10 +9,18 @@ defmodule Plausible.Stats.TableDecider do alias Plausible.Stats.Query def events_join_sessions?(query) do - Enum.any?(query.filters, &(filters_partitioner(query, &1) == :session)) + query + |> filter_keys() + |> Enum.any?(&(filters_partitioner(query, &1) == :session)) end - def partition_metrics(metrics, query, breakdown_property \\ nil) do + def event_filters?(query) do + query + |> filter_keys() + |> Enum.any?(&(filters_partitioner(query, &1) == :event)) + end + + def partition_metrics(metrics, query) do %{ event: event_only_metrics, session: session_only_metrics, @@ -22,16 +30,10 @@ defmodule Plausible.Stats.TableDecider do } = partition(metrics, query, &metric_partitioner/2) - # Treat breakdown property as yet another filter - query = - if breakdown_property do - Query.put_filter(query, [:is, breakdown_property, []]) - else - query - end - %{event: event_only_filters, session: session_only_filters} = - partition(query.filters, query, &filters_partitioner/2) + query + |> filter_keys() + |> partition(query, &filters_partitioner/2) cond do # Only one table needs to be queried @@ -55,6 +57,12 @@ defmodule Plausible.Stats.TableDecider do end end + defp filter_keys(query) do + query.filters + |> Enum.map(fn [_, filter_key | _rest] -> filter_key end) + |> Enum.concat(query.dimensions) + end + defp metric_partitioner(_, :conversion_rate), do: :event defp metric_partitioner(_, :average_revenue), do: :event defp metric_partitioner(_, :total_revenue), do: :event @@ -83,16 +91,16 @@ defmodule Plausible.Stats.TableDecider do defp metric_partitioner(_, _), do: :either - defp filters_partitioner(_, [_, "event:" <> _ | _rest]), do: :event - defp filters_partitioner(_, [_, "visit:entry_page" | _rest]), do: :session - defp filters_partitioner(_, [_, "visit:entry_page_hostname" | _rest]), do: :session - defp filters_partitioner(_, [_, "visit:exit_page" | _rest]), do: :session - defp filters_partitioner(_, [_, "visit:exit_page_hostname" | _rest]), do: :session + defp filters_partitioner(_, "event:" <> _), do: :event + defp filters_partitioner(_, "visit:entry_page"), do: :session + defp filters_partitioner(_, "visit:entry_page_hostname"), do: :session + defp filters_partitioner(_, "visit:exit_page"), do: :session + defp filters_partitioner(_, "visit:exit_page_hostname"), do: :session - defp filters_partitioner(%Query{experimental_reduced_joins?: true}, [_, "visit:" <> _ | _rest]), + defp filters_partitioner(%Query{experimental_reduced_joins?: true}, "visit:" <> _), do: :either - defp filters_partitioner(_, [_, "visit:" <> _ | _rest]), + defp filters_partitioner(_, "visit:" <> _), do: :session defp filters_partitioner(%Query{experimental_reduced_joins?: false}, {unknown, _}) do diff --git a/lib/plausible_web/controllers/api/external_query_api_controller.ex b/lib/plausible_web/controllers/api/external_query_api_controller.ex new file mode 100644 index 000000000..d2bf01401 --- /dev/null +++ b/lib/plausible_web/controllers/api/external_query_api_controller.ex @@ -0,0 +1,23 @@ +defmodule PlausibleWeb.Api.ExternalQueryApiController do + @moduledoc false + + use PlausibleWeb, :controller + use Plausible.Repo + use PlausibleWeb.Plugs.ErrorHandler + alias Plausible.Stats.Query + + def query(conn, params) do + site = Repo.preload(conn.assigns.site, :owner) + + case Query.build(site, params) do + {:ok, query} -> + results = Plausible.Stats.query(site, query) + json(conn, results) + + {:error, message} -> + conn + |> put_status(400) + |> json(%{error: message}) + end + end +end diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex index ab83484c1..bd7179c13 100644 --- a/lib/plausible_web/router.ex +++ b/lib/plausible_web/router.ex @@ -175,6 +175,12 @@ defmodule PlausibleWeb.Router do get "/timeseries", ExternalStatsController, :timeseries end + scope "/api/v2", PlausibleWeb.Api do + pipe_through [:public_api, PlausibleWeb.AuthorizeStatsApiPlug] + + post "/query", ExternalQueryApiController, :query + end + on_ee do scope "/api/v1/sites", PlausibleWeb.Api do pipe_through [:public_api, PlausibleWeb.AuthorizeSitesApiPlug] diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exs index 39baead3f..481f0362d 100644 --- a/test/plausible/stats/query_parser_test.exs +++ b/test/plausible/stats/query_parser_test.exs @@ -1,44 +1,50 @@ defmodule Plausible.Stats.Filters.QueryParserTest do - use ExUnit.Case, async: true + use Plausible.DataCase + alias Plausible.Stats.Filters import Plausible.Stats.Filters.QueryParser - def check_success(params, expected_result) do - assert parse(params) == {:ok, expected_result} + setup [:create_user, :create_new_site] + + @date_range Date.range(Timex.today(), Timex.today()) + + def check_success(params, site, expected_result) do + assert parse(site, params) == {:ok, expected_result} end - def check_error(params, expected_error_message) do - {:error, message} = parse(params) + def check_error(params, site, expected_error_message) do + {:error, message} = parse(site, params) assert message =~ expected_error_message end - test "parsing empty map fails" do + test "parsing empty map fails", %{site: site} do %{} - |> check_error("No valid metrics passed") + |> check_error(site, "No valid metrics passed") end describe "metrics validation" do - test "valid metrics passed" do + test "valid metrics passed", %{site: site} do %{"metrics" => ["visitors", "events"], "date_range" => "all"} - |> check_success(%{ + |> check_success(site, %{ metrics: [:visitors, :events], - date_range: "all", + date_range: @date_range, filters: [], dimensions: [], - order_by: nil + order_by: nil, + timezone: site.timezone, + imported_data_requested: false }) end - test "invalid metric passed" do + test "invalid metric passed", %{site: site} do %{"metrics" => ["visitors", "event:name"], "date_range" => "all"} - |> check_error("Unknown metric '\"event:name\"'") + |> check_error(site, "Unknown metric '\"event:name\"'") end - test "fuller list of metrics" do + test "fuller list of metrics", %{site: site} do %{ "metrics" => [ "time_on_page", - "conversion_rate", "visitors", "pageviews", "visits", @@ -48,10 +54,9 @@ defmodule Plausible.Stats.Filters.QueryParserTest do ], "date_range" => "all" } - |> check_success(%{ + |> check_success(site, %{ metrics: [ :time_on_page, - :conversion_rate, :visitors, :pageviews, :visits, @@ -59,22 +64,24 @@ defmodule Plausible.Stats.Filters.QueryParserTest do :bounce_rate, :visit_duration ], - date_range: "all", + date_range: @date_range, filters: [], dimensions: [], - order_by: nil + order_by: nil, + timezone: site.timezone, + imported_data_requested: false }) end - test "same metric queried multiple times" do + test "same metric queried multiple times", %{site: site} do %{"metrics" => ["events", "visitors", "visitors"], "date_range" => "all"} - |> check_error(~r/Metrics cannot be queried multiple times/) + |> check_error(site, ~r/Metrics cannot be queried multiple times/) end end describe "filters validation" do for operation <- [:is, :is_not, :matches, :does_not_match] do - test "#{operation} filter" do + test "#{operation} filter", %{site: site} do %{ "metrics" => ["visitors"], "date_range" => "all", @@ -82,18 +89,20 @@ defmodule Plausible.Stats.Filters.QueryParserTest do [Atom.to_string(unquote(operation)), "event:name", ["foo"]] ] } - |> check_success(%{ + |> check_success(site, %{ metrics: [:visitors], - date_range: "all", + date_range: @date_range, filters: [ [unquote(operation), "event:name", ["foo"]] ], dimensions: [], - order_by: nil + order_by: nil, + timezone: site.timezone, + imported_data_requested: false }) end - test "#{operation} filter with invalid clause" do + test "#{operation} filter with invalid clause", %{site: site} do %{ "metrics" => ["visitors"], "date_range" => "all", @@ -101,11 +110,11 @@ defmodule Plausible.Stats.Filters.QueryParserTest do [Atom.to_string(unquote(operation)), "event:name", "foo"] ] } - |> check_error(~r/Invalid filter/) + |> check_error(site, ~r/Invalid filter/) end end - test "filtering by invalid operation" do + test "filtering by invalid operation", %{site: site} do %{ "metrics" => ["visitors"], "date_range" => "all", @@ -113,10 +122,10 @@ defmodule Plausible.Stats.Filters.QueryParserTest do ["exists?", "event:name", ["foo"]] ] } - |> check_error(~r/Unknown operator for filter/) + |> check_error(site, ~r/Unknown operator for filter/) end - test "filtering by custom properties" do + test "filtering by custom properties", %{site: site} do %{ "metrics" => ["visitors"], "date_range" => "all", @@ -124,20 +133,22 @@ defmodule Plausible.Stats.Filters.QueryParserTest do ["is", "event:props:foobar", ["value"]] ] } - |> check_success(%{ + |> check_success(site, %{ metrics: [:visitors], - date_range: "all", + date_range: @date_range, filters: [ [:is, "event:props:foobar", ["value"]] ], dimensions: [], - order_by: nil + order_by: nil, + timezone: site.timezone, + imported_data_requested: false }) end for dimension <- Filters.event_props() do if dimension != "goal" do - test "filtering by event:#{dimension} filter" do + test "filtering by event:#{dimension} filter", %{site: site} do %{ "metrics" => ["visitors"], "date_range" => "all", @@ -145,21 +156,23 @@ defmodule Plausible.Stats.Filters.QueryParserTest do ["is", "event:#{unquote(dimension)}", ["foo"]] ] } - |> check_success(%{ + |> check_success(site, %{ metrics: [:visitors], - date_range: "all", + date_range: @date_range, filters: [ [:is, "event:#{unquote(dimension)}", ["foo"]] ], dimensions: [], - order_by: nil + order_by: nil, + timezone: site.timezone, + imported_data_requested: false }) end end end for dimension <- Filters.visit_props() do - test "filtering by visit:#{dimension} filter" do + test "filtering by visit:#{dimension} filter", %{site: site} do %{ "metrics" => ["visitors"], "date_range" => "all", @@ -167,38 +180,21 @@ defmodule Plausible.Stats.Filters.QueryParserTest do ["is", "visit:#{unquote(dimension)}", ["foo"]] ] } - |> check_success(%{ + |> check_success(site, %{ metrics: [:visitors], - date_range: "all", + date_range: @date_range, filters: [ [:is, "visit:#{unquote(dimension)}", ["foo"]] ], dimensions: [], - order_by: nil + order_by: nil, + timezone: site.timezone, + imported_data_requested: false }) end end - test "filtering by event:goal" do - %{ - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [ - ["is", "event:goal", ["Signup", "Visit /thank-you"]] - ] - } - |> check_success(%{ - metrics: [:visitors], - date_range: "all", - filters: [ - [:is, "event:goal", [{:event, "Signup"}, {:page, "/thank-you"}]] - ], - dimensions: [], - order_by: nil - }) - end - - test "invalid event filter" do + test "invalid event filter", %{site: site} do %{ "metrics" => ["visitors"], "date_range" => "all", @@ -206,10 +202,10 @@ defmodule Plausible.Stats.Filters.QueryParserTest do ["is", "event:device", ["foo"]] ] } - |> check_error(~r/Invalid filter /) + |> check_error(site, ~r/Invalid filter /) end - test "invalid visit filter" do + test "invalid visit filter", %{site: site} do %{ "metrics" => ["visitors"], "date_range" => "all", @@ -217,16 +213,92 @@ defmodule Plausible.Stats.Filters.QueryParserTest do ["is", "visit:name", ["foo"]] ] } - |> check_error(~r/Invalid filter /) + |> check_error(site, ~r/Invalid filter /) end - test "invalid filter" do + test "invalid filter", %{site: site} do %{ "metrics" => ["visitors"], "date_range" => "all", "filters" => "foobar" } - |> check_error(~r/Invalid filters passed/) + |> check_error(site, ~r/Invalid filters passed/) + end + end + + describe "include validation" do + test "setting include.imports", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "include" => %{"imports" => true} + } + |> check_success(site, %{ + metrics: [:visitors], + date_range: @date_range, + filters: [], + dimensions: [], + order_by: nil, + timezone: site.timezone, + imported_data_requested: true + }) + end + + test "setting invalid imports value", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "include" => "foobar" + } + |> check_error(site, ~r/Invalid include passed/) + end + end + + describe "event:goal filter validation" do + test "valid filters", %{site: site} do + insert(:goal, %{site: site, event_name: "Signup"}) + insert(:goal, %{site: site, page_path: "/thank-you"}) + + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["is", "event:goal", ["Signup", "Visit /thank-you"]] + ] + } + |> check_success(site, %{ + metrics: [:visitors], + date_range: @date_range, + filters: [ + [:is, "event:goal", [{:event, "Signup"}, {:page, "/thank-you"}]] + ], + dimensions: [], + order_by: nil, + timezone: site.timezone, + imported_data_requested: false + }) + end + + test "invalid event filter", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["is", "event:goal", ["Signup"]] + ] + } + |> check_error(site, ~r/The goal `Signup` is not configured for this site/) + end + + test "invalid page filter", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["is", "event:goal", ["Visit /thank-you"]] + ] + } + |> check_error(site, ~r/The goal `Visit \/thank-you` is not configured for this site/) end end @@ -235,139 +307,288 @@ defmodule Plausible.Stats.Filters.QueryParserTest do describe "dimensions validation" do for dimension <- Filters.event_props() do - test "event:#{dimension} dimension" do + test "event:#{dimension} dimension", %{site: site} do %{ "metrics" => ["visitors"], "date_range" => "all", "dimensions" => ["event:#{unquote(dimension)}"] } - |> check_success(%{ + |> check_success(site, %{ metrics: [:visitors], - date_range: "all", + date_range: @date_range, filters: [], dimensions: ["event:#{unquote(dimension)}"], - order_by: nil + order_by: nil, + timezone: site.timezone, + imported_data_requested: false }) end end for dimension <- Filters.visit_props() do - test "visit:#{dimension} dimension" do + test "visit:#{dimension} dimension", %{site: site} do %{ "metrics" => ["visitors"], "date_range" => "all", "dimensions" => ["visit:#{unquote(dimension)}"] } - |> check_success(%{ + |> check_success(site, %{ metrics: [:visitors], - date_range: "all", + date_range: @date_range, filters: [], dimensions: ["visit:#{unquote(dimension)}"], - order_by: nil + order_by: nil, + timezone: site.timezone, + imported_data_requested: false }) end end - test "custom properties dimension" do + test "custom properties dimension", %{site: site} do %{ "metrics" => ["visitors"], "date_range" => "all", "dimensions" => ["event:props:foobar"] } - |> check_success(%{ + |> check_success(site, %{ metrics: [:visitors], - date_range: "all", + date_range: @date_range, filters: [], dimensions: ["event:props:foobar"], - order_by: nil + order_by: nil, + timezone: site.timezone, + imported_data_requested: false }) end - test "invalid dimension name passed" do + test "invalid custom property dimension", %{site: site} do + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:"] + } + |> check_error(site, ~r/Invalid dimensions/) + end + + test "invalid dimension name passed", %{site: site} do %{ "metrics" => ["visitors"], "date_range" => "all", "dimensions" => ["visitors"] } - |> check_error(~r/Invalid dimensions/) + |> check_error(site, ~r/Invalid dimensions/) end - test "invalid dimension" do + test "invalid dimension", %{site: site} do %{ "metrics" => ["visitors"], "date_range" => "all", "dimensions" => "foobar" } - |> check_error(~r/Invalid dimensions/) + |> check_error(site, ~r/Invalid dimensions/) end - test "dimensions are not unique" do + test "dimensions are not unique", %{site: site} do %{ "metrics" => ["visitors"], "date_range" => "all", "dimensions" => ["event:name", "event:name"] } - |> check_error(~r/Some dimensions are listed multiple times/) + |> check_error(site, ~r/Some dimensions are listed multiple times/) end end describe "order_by validation" do - test "ordering by metric" do + test "ordering by metric", %{site: site} do %{ "metrics" => ["visitors", "events"], "date_range" => "all", "order_by" => [["events", "desc"], ["visitors", "asc"]] } - |> check_success(%{ + |> check_success(site, %{ metrics: [:visitors, :events], - date_range: "all", + date_range: @date_range, filters: [], dimensions: [], - order_by: [{:events, :desc}, {:visitors, :asc}] + order_by: [{:events, :desc}, {:visitors, :asc}], + timezone: site.timezone, + imported_data_requested: false }) end - test "ordering by dimension" do + test "ordering by dimension", %{site: site} do %{ "metrics" => ["visitors"], "date_range" => "all", "dimensions" => ["event:name"], "order_by" => [["event:name", "desc"]] } - |> check_success(%{ + |> check_success(site, %{ metrics: [:visitors], - date_range: "all", + date_range: @date_range, filters: [], dimensions: ["event:name"], - order_by: [{"event:name", :desc}] + order_by: [{"event:name", :desc}], + timezone: site.timezone, + imported_data_requested: false }) end - test "ordering by invalid value" do + test "ordering by invalid value", %{site: site} do %{ "metrics" => ["visitors"], "date_range" => "all", "order_by" => [["visssss", "desc"]] } - |> check_error(~r/Invalid order_by entry/) + |> check_error(site, ~r/Invalid order_by entry/) end - test "ordering by not queried metric" do + test "ordering by not queried metric", %{site: site} do %{ "metrics" => ["visitors"], "date_range" => "all", "order_by" => [["events", "desc"]] } - |> check_error(~r/Entry is not a queried metric or dimension/) + |> check_error(site, ~r/Entry is not a queried metric or dimension/) end - test "ordering by not queried dimension" do + test "ordering by not queried dimension", %{site: site} do %{ "metrics" => ["visitors"], "date_range" => "all", "order_by" => [["event:name", "desc"]] } - |> check_error(~r/Entry is not a queried metric or dimension/) + |> check_error(site, ~r/Entry is not a queried metric or dimension/) + end + end + + describe "custom props access" do + test "error if invalid filter", %{site: site, user: user} do + ep = + insert(:enterprise_plan, features: [Plausible.Billing.Feature.StatsAPI], user_id: user.id) + + insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id) + + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["is", "event:props:foobar", ["foo"]]] + } + |> check_error( + site, + ~r/The owner of this site does not have access to the custom properties feature/ + ) + end + + test "error if invalid dimension", %{site: site, user: user} do + ep = + insert(:enterprise_plan, features: [Plausible.Billing.Feature.StatsAPI], user_id: user.id) + + insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id) + + %{ + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:foobar"] + } + |> check_error( + site, + ~r/The owner of this site does not have access to the custom properties feature/ + ) + end + end + + describe "conversion_rate metric" do + test "fails validation on its own", %{site: site} do + %{ + "metrics" => ["conversion_rate"], + "date_range" => "all" + } + |> check_error( + site, + ~r/Metric `conversion_rate` can only be queried with event:goal filters or dimensions/ + ) + end + + test "succeeds with event:goal filter", %{site: site} do + insert(:goal, %{site: site, event_name: "Signup"}) + + %{ + "metrics" => ["conversion_rate"], + "date_range" => "all", + "filters" => [["is", "event:goal", ["Signup"]]] + } + |> check_success(site, %{ + metrics: [:conversion_rate], + date_range: @date_range, + filters: [[:is, "event:goal", [event: "Signup"]]], + dimensions: [], + order_by: nil, + timezone: site.timezone, + imported_data_requested: false + }) + end + + test "succeeds with event:goal dimension", %{site: site} do + insert(:goal, %{site: site, event_name: "Signup"}) + + %{ + "metrics" => ["conversion_rate"], + "date_range" => "all", + "dimensions" => ["event:goal"] + } + |> check_success(site, %{ + metrics: [:conversion_rate], + date_range: @date_range, + filters: [], + dimensions: ["event:goal"], + order_by: nil, + timezone: site.timezone, + imported_data_requested: false + }) + end + end + + describe "session metrics" do + test "single session metric succeeds", %{site: site} do + %{ + "metrics" => ["bounce_rate"], + "date_range" => "all", + "dimensions" => ["visit:device"] + } + |> check_success(site, %{ + metrics: [:bounce_rate], + date_range: @date_range, + filters: [], + dimensions: ["visit:device"], + order_by: nil, + timezone: site.timezone, + imported_data_requested: false + }) + end + + test "fails if using session metric with event dimension", %{site: site} do + %{ + "metrics" => ["bounce_rate"], + "date_range" => "all", + "dimensions" => ["event:props:foo"] + } + |> check_error( + site, + "Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions" + ) + end + + test "fails if using session metric with event filter", %{site: site} do + %{ + "metrics" => ["bounce_rate"], + "date_range" => "all", + "filters" => [["is", "event:props:foo", ["(none)"]]] + } + |> check_error( + site, + "Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions" + ) end end end diff --git a/test/plausible/stats/table_decider_test.exs b/test/plausible/stats/table_decider_test.exs index c7d85bbe7..7226542c1 100644 --- a/test/plausible/stats/table_decider_test.exs +++ b/test/plausible/stats/table_decider_test.exs @@ -5,62 +5,62 @@ defmodule Plausible.Stats.TableDeciderTest do import Plausible.Stats.TableDecider test "events_join_sessions? with experimental_reduced_joins disabled" do - assert not events_join_sessions?(make_query(false, %{})) - assert not events_join_sessions?(make_query(false, %{name: "pageview"})) - assert events_join_sessions?(make_query(false, %{source: "Google"})) - assert events_join_sessions?(make_query(false, %{entry_page: "/"})) - assert events_join_sessions?(make_query(false, %{exit_page: "/"})) + assert not events_join_sessions?(make_query(false, [])) + assert not events_join_sessions?(make_query(false, ["event:name"])) + assert events_join_sessions?(make_query(false, ["visit:source"])) + assert events_join_sessions?(make_query(false, ["visit:entry_page"])) + assert events_join_sessions?(make_query(false, ["visit:exit_page"])) end test "events_join_sessions? with experimental_reduced_joins enabled" do - assert not events_join_sessions?(make_query(true, %{})) - assert not events_join_sessions?(make_query(true, %{name: "pageview"})) - assert not events_join_sessions?(make_query(true, %{source: "Google"})) - assert events_join_sessions?(make_query(true, %{entry_page: "/"})) - assert events_join_sessions?(make_query(true, %{exit_page: "/"})) + assert not events_join_sessions?(make_query(true, [])) + assert not events_join_sessions?(make_query(true, ["event:name"])) + assert not events_join_sessions?(make_query(true, ["visit:source"])) + assert events_join_sessions?(make_query(true, ["visit:entry_page"])) + assert events_join_sessions?(make_query(true, ["visit:exit_page"])) end describe "partition_metrics" do test "with no metrics or filters" do - query = make_query(false, %{}) + query = make_query(false, []) assert partition_metrics([], query) == {[], [], []} end test "session-only metrics accordingly" do - query = make_query(false, %{}) + query = make_query(false, []) assert partition_metrics([:bounce_rate, :views_per_visit], query) == {[], [:bounce_rate, :views_per_visit], []} end test "event-only metrics accordingly" do - query = make_query(false, %{}) + query = make_query(false, []) assert partition_metrics([:total_revenue, :visitors], query) == {[:total_revenue, :visitors], [], []} end test "filters from both, event-only metrics" do - query = make_query(false, %{name: "pageview", source: "Google"}) + query = make_query(false, ["event:name", "visit:source"]) assert partition_metrics([:total_revenue], query) == {[:total_revenue], [], []} end test "filters from both, session-only metrics" do - query = make_query(false, %{name: "pageview", source: "Google"}) + query = make_query(false, ["event:name", "visit:source"]) assert partition_metrics([:bounce_rate], query) == {[], [:bounce_rate], []} end test "session filters but no session metrics" do - query = make_query(false, %{source: "Google"}) + query = make_query(false, ["visit:source"]) assert partition_metrics([:total_revenue], query) == {[:total_revenue], [], []} end test "sample_percent is added to both types of metrics" do - query = make_query(false, %{}) + query = make_query(false, []) assert partition_metrics([:total_revenue, :sample_percent], query) == {[:total_revenue, :sample_percent], [], []} @@ -73,14 +73,14 @@ defmodule Plausible.Stats.TableDeciderTest do end test "other metrics put in its own result" do - query = make_query(false, %{}) + query = make_query(false, []) assert partition_metrics([:time_on_page, :percentage, :total_visitors], query) == {[], [], [:time_on_page, :percentage, :total_visitors]} end test "raises if unknown metric" do - query = make_query(false, %{}) + query = make_query(false, []) assert_raise ArgumentError, fn -> partition_metrics([:foobar], query) @@ -90,7 +90,7 @@ defmodule Plausible.Stats.TableDeciderTest do describe "partition_metrics with experimental_reduced_joins enabled" do test "metrics that can be calculated on either when event-only metrics" do - query = make_query(true, %{}) + query = make_query(true, []) assert partition_metrics([:total_revenue, :visitors], query) == {[:total_revenue, :visitors], [], []} @@ -99,7 +99,7 @@ defmodule Plausible.Stats.TableDeciderTest do end test "metrics that can be calculated on either when session-only metrics" do - query = make_query(true, %{}) + query = make_query(true, []) assert partition_metrics([:bounce_rate, :visitors], query) == {[], [:bounce_rate, :visitors], []} @@ -109,56 +109,60 @@ defmodule Plausible.Stats.TableDeciderTest do end test "metrics that can be calculated on either are biased to sessions" do - query = make_query(true, %{}) + query = make_query(true, []) assert partition_metrics([:bounce_rate, :total_revenue, :visitors], query) == {[:total_revenue], [:bounce_rate, :visitors], []} end test "sample_percent is handled with either metrics" do - query = make_query(true, %{}) + query = make_query(true, []) assert partition_metrics([:visitors, :sample_percent], query) == {[], [:visitors, :sample_percent], []} end test "metric can be calculated on either, but filtering on events" do - query = make_query(true, %{name: "pageview"}) + query = make_query(true, ["event:name"]) assert partition_metrics([:visitors], query) == {[:visitors], [], []} end test "metric can be calculated on either, but filtering on events and sessions" do - query = make_query(true, %{name: "pageview", exit_page: "/"}) + query = make_query(true, ["event:name", "visit:exit_page"]) assert partition_metrics([:visitors], query) == {[], [:visitors], []} end test "metric can be calculated on either, filtering on either" do - query = make_query(true, %{source: "Google"}) + query = make_query(true, ["visit:source"]) assert partition_metrics([:visitors], query) == {[], [:visitors], []} end test "metric can be calculated on either, filtering on sessions" do - query = make_query(true, %{exit_page: "/"}) + query = make_query(true, ["visit:exit_page"]) assert partition_metrics([:visitors], query) == {[], [:visitors], []} end - test "breakdown value leans metric" do - query = make_query(true, %{}) + test "query dimensions lean metric" do + assert partition_metrics([:visitors], make_query(true, [], ["event:name"])) == + {[:visitors], [], []} - assert partition_metrics([:visitors], query, "event:name") == {[:visitors], [], []} - assert partition_metrics([:visitors], query, "visit:source") == {[], [:visitors], []} - assert partition_metrics([:visitors], query, "visit:exit_page") == {[], [:visitors], []} + assert partition_metrics([:visitors], make_query(true, [], ["visit:source"])) == + {[], [:visitors], []} + + assert partition_metrics([:visitors], make_query(true, [], ["visit:exit_page"])) == + {[], [:visitors], []} end end - defp make_query(experimental_reduced_joins?, filters) do + defp make_query(experimental_reduced_joins?, filter_keys, dimensions \\ []) do Query.from(build(:site), %{ "experimental_reduced_joins" => to_string(experimental_reduced_joins?), - "filters" => Jason.encode!(filters) + "filters" => Enum.map(filter_keys, fn filter_key -> ["is", filter_key, []] end), + "dimensions" => dimensions }) end end diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs new file mode 100644 index 000000000..fa6ab457f --- /dev/null +++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs @@ -0,0 +1,3106 @@ +defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do + use PlausibleWeb.ConnCase + alias Plausible.Billing.Feature + + @user_id 1231 + + setup [:create_user, :create_new_site, :create_api_key, :use_api_key] + + describe "feature access" do + test "cannot break down by a custom prop without access to the props feature", %{ + conn: conn, + user: user, + site: site + } do + ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id) + insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:author"] + }) + + assert json_response(conn, 400)["error"] == + "The owner of this site does not have access to the custom properties feature" + end + + test "can break down by an internal prop key without access to the props feature", %{ + conn: conn, + user: user, + site: site + } do + ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id) + insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:path"] + }) + + assert json_response(conn, 200)["results"] + end + + test "cannot filter by a custom prop without access to the props feature", %{ + conn: conn, + user: user, + site: site + } do + ep = + insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id) + + insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:source"], + "filters" => [["is", "event:props:author", ["Uku"]]] + }) + + assert json_response(conn, 400)["error"] == + "The owner of this site does not have access to the custom properties feature" + end + + test "can filter by an internal prop key without access to the props feature", %{ + conn: conn, + user: user, + site: site + } do + ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id) + insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:source"], + "filters" => [["is", "event:props:url", ["whatever"]]] + }) + + assert json_response(conn, 200)["results"] + end + end + + describe "param validation" do + test "does not allow querying conversion_rate without a goal filter", %{ + conn: conn, + site: site + } do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["conversion_rate"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [["is", "event:props:author", ["Uku"]]] + }) + + assert json_response(conn, 400)["error"] == + "Metric `conversion_rate` can only be queried with event:goal filters or dimensions" + end + + test "validates that dimensions are valid", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["badproperty"] + }) + + assert json_response(conn, 400)["error"] =~ "Invalid dimensions" + end + + test "empty custom property is invalid", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:"] + }) + + assert json_response(conn, 400)["error"] =~ "Invalid dimensions" + end + + test "validates that correct date range is used", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "bad_period", + "dimensions" => ["event:name"] + }) + + assert json_response(conn, 400)["error"] =~ "Invalid date range" + end + + test "fails when an invalid metric is provided", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "baa"], + "date_range" => "all", + "dimensions" => ["event:name"] + }) + + assert json_response(conn, 400)["error"] =~ "Unknown metric '\"baa\"'" + end + + test "session metrics cannot be used with event:name property", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "bounce_rate"], + "date_range" => "all", + "dimensions" => ["event:name"] + }) + + assert json_response(conn, 400)["error"] =~ + "Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions" + end + + test "session metrics cannot be used with event:props:* property", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "bounce_rate"], + "date_range" => "all", + "dimensions" => ["event:props:url"] + }) + + assert json_response(conn, 400)["error"] =~ + "Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions" + end + + test "session metrics cannot be used with event:name filter", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "bounce_rate"], + "date_range" => "all", + "dimensions" => ["visit:device"], + "filters" => [ + ["is", "event:name", ["pageview"]] + ] + }) + + assert json_response(conn, 400)["error"] =~ + "Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions" + end + end + + test "breakdown by visit:source", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + referrer_source: "Google", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + referrer_source: "Google", + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, + referrer_source: "", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:source"] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Google"], "metrics" => [2]}, + %{"dimensions" => ["Direct / None"], "metrics" => [1]} + ] + end + + test "breakdown by visit:country", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, country_code: "EE", timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, country_code: "EE", timestamp: ~N[2021-01-01 00:25:00]), + build(:pageview, country_code: "US", timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:country"] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["EE"], "metrics" => [2]}, + %{"dimensions" => ["US"], "metrics" => [1]} + ] + end + + # test "breaks down all metrics by visit:referrer with imported data", %{conn: conn, site: site} do + # site_import = + # insert(:site_import, + # site: site, + # start_date: ~D[2005-01-01], + # end_date: Timex.today(), + # source: :universal_analytics + # ) + + # populate_stats(site, site_import.id, [ + # build(:pageview, referrer: "site.com", timestamp: ~N[2021-01-01 00:00:00]), + # build(:pageview, referrer: "site.com/1", timestamp: ~N[2021-01-01 00:00:00]), + # build(:imported_sources, + # referrer: "site.com", + # date: ~D[2021-01-01], + # visitors: 2, + # visits: 2, + # pageviews: 2, + # bounces: 1, + # visit_duration: 120 + # ), + # build(:imported_sources, + # referrer: "site.com/2", + # date: ~D[2021-01-01], + # visitors: 2, + # visits: 2, + # pageviews: 2, + # bounces: 2, + # visit_duration: 0 + # ), + # build(:imported_sources, + # date: ~D[2021-01-01], + # visitors: 10, + # visits: 11, + # pageviews: 50, + # bounces: 0, + # visit_duration: 1100 + # ) + # ]) + + # conn = + # get(conn, "/api/v1/stats/breakdown", %{ + # "site_id" => site.domain, + # "period" => "day", + # "metrics" => "visitors,visits,pageviews,bounce_rate,visit_duration", + # "date" => "2021-01-01", + # "property" => "visit:referrer", + # "with_imported" => "true" + # }) + + # assert json_response(conn, 200) == %{ + # "results" => [ + # %{ + # "referrer" => "Direct / None", + # "visitors" => 10, + # "visits" => 11, + # "pageviews" => 50, + # "bounce_rate" => 0, + # "visit_duration" => 100 + # }, + # %{ + # "referrer" => "site.com", + # "visitors" => 3, + # "visits" => 3, + # "pageviews" => 3, + # "bounce_rate" => 67.0, + # "visit_duration" => 40 + # }, + # %{ + # "referrer" => "site.com/2", + # "visitors" => 2, + # "visits" => 2, + # "pageviews" => 2, + # "bounce_rate" => 100.0, + # "visit_duration" => 0 + # }, + # %{ + # "referrer" => "site.com/1", + # "visitors" => 1, + # "visits" => 1, + # "pageviews" => 1, + # "bounce_rate" => 100.0, + # "visit_duration" => 0 + # } + # ] + # } + + # conn = + # post(conn, "/api/v2/query", %{ + # "site_id" => site.domain, + # "metrics" => ["visitors", "visits", "pageviews", "bounce_rate", "visit_duration"], + # "date_range" => "all", + # "dimensions" => ["visit:referrer"], + # }) + + # %{"results" => results} = json_response(conn, 200) + + # assert results == [ + # %{"dimensions" => ["EE"], "metrics" => [2]}, + # %{"dimensions" => ["US"], "metrics" => [1]}, + # ] + # end + + # for {property, attr} <- [ + # {"visit:utm_campaign", :utm_campaign}, + # {"visit:utm_source", :utm_source}, + # {"visit:utm_term", :utm_term}, + # {"visit:utm_content", :utm_content} + # ] do + # test "breakdown by #{property} when filtered by hostname", %{conn: conn, site: site} do + # populate_stats(site, [ + # # session starts at two.example.com with utm_param=ad + # build( + # :pageview, + # [ + # {unquote(attr), "ad"}, + # {:user_id, @user_id}, + # {:hostname, "two.example.com"}, + # {:timestamp, ~N[2021-01-01 00:00:00]} + # ] + # ), + # # session continues on one.example.com without any utm_params + # build( + # :pageview, + # [ + # {:user_id, @user_id}, + # {:hostname, "one.example.com"}, + # {:timestamp, ~N[2021-01-01 00:15:00]} + # ] + # ) + # ]) + + # conn = + # get(conn, "/api/v1/stats/breakdown", %{ + # "site_id" => site.domain, + # "period" => "day", + # "date" => "2021-01-01", + # "filters" => "event:hostname==one.example.com", + # "property" => unquote(property) + # }) + + # # nobody landed on one.example.com from utm_param=ad + # assert json_response(conn, 200) == %{"results" => []} + # end + # end + + for {dimension, column, value1, value2, blank_value} <- [ + {"visit:source", :referrer_source, "Google", "Twitter", "Direct / None"}, + {"visit:referrer", :referrer, "example.com", "google.com", "Direct / None"}, + {"visit:utm_medium", :utm_medium, "Search", "social", "(not set)"}, + {"visit:utm_source", :utm_source, "Google", "Bing", "(not set)"}, + {"visit:utm_campaign", :utm_campaign, "ads", "profile", "(not set)"}, + {"visit:utm_content", :utm_content, "Content1", "blog2", "(not set)"}, + {"visit:utm_term", :utm_term, "Term1", "favicon", "(not set)"}, + {"visit:os", :operating_system, "Mac", "Windows", "(not set)"}, + {"visit:browser", :browser, "Chrome", "Safari", "(not set)"}, + {"visit:device", :screen_size, "Mobile", "Desktop", "(not set)"} + ] do + test "simple breakdown by #{dimension}", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, [ + {unquote(column), unquote(value1)}, + {:timestamp, ~N[2021-01-01 00:00:00]} + ]), + build(:pageview, [ + {unquote(column), unquote(value1)}, + {:timestamp, ~N[2021-01-01 00:25:00]} + ]), + build(:pageview, [ + {unquote(column), unquote(value1)}, + {:timestamp, ~N[2021-01-01 00:55:00]} + ]), + build(:pageview, [ + {unquote(column), unquote(value2)}, + {:timestamp, ~N[2021-01-01 01:00:00]} + ]), + build(:pageview, [ + {unquote(column), unquote(value2)}, + {:timestamp, ~N[2021-01-01 01:25:00]} + ]), + build(:pageview, [ + {unquote(column), ""}, + {:timestamp, ~N[2021-01-01 00:00:00]} + ]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => ["2021-01-01", "2021-01-01"], + "dimensions" => [unquote(dimension)] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => [unquote(value1)], "metrics" => [3]}, + %{"dimensions" => [unquote(value2)], "metrics" => [2]}, + %{"dimensions" => [unquote(blank_value)], "metrics" => [1]} + ] + end + end + + test "breakdown by visit:os and visit:os_version", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, operating_system: "Mac", operating_system_version: "14"), + build(:pageview, operating_system: "Mac", operating_system_version: "14"), + build(:pageview, operating_system: "Mac", operating_system_version: "14"), + build(:pageview, operating_system_version: "14"), + build(:pageview, + operating_system: "Windows", + operating_system_version: "11" + ), + build(:pageview, + operating_system: "Windows", + operating_system_version: "11" + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:os", "visit:os_version"] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Mac", "14"], "metrics" => [3]}, + %{"dimensions" => ["Windows", "11"], "metrics" => [2]}, + %{"dimensions" => ["(not set)", "14"], "metrics" => [1]} + ] + end + + test "breakdown by visit:browser and visit:browser_version", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, browser: "Chrome", browser_version: "14"), + build(:pageview, browser: "Chrome", browser_version: "14"), + build(:pageview, browser: "Chrome", browser_version: "14"), + build(:pageview, browser_version: "14"), + build(:pageview, + browser: "Firefox", + browser_version: "11" + ), + build(:pageview, + browser: "Firefox", + browser_version: "11" + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:browser", "visit:browser_version"] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Chrome", "14"], "metrics" => [3]}, + %{"dimensions" => ["Firefox", "11"], "metrics" => [2]}, + %{"dimensions" => ["(not set)", "14"], "metrics" => [1]} + ] + end + + test "explicit order_by", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, operating_system: "Windows", browser: "Chrome"), + build(:pageview, operating_system: "Windows", browser: "Firefox"), + build(:pageview, operating_system: "Linux", browser: "Firefox"), + build(:pageview, operating_system: "Linux", browser: "Firefox"), + build(:pageview, operating_system: "Mac", browser: "Chrome") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:os", "visit:browser"], + "order_by" => [["visitors", "asc"], ["visit:browser", "desc"], ["visit:os", "asc"]] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Windows", "Firefox"], "metrics" => [1]}, + %{"dimensions" => ["Mac", "Chrome"], "metrics" => [1]}, + %{"dimensions" => ["Windows", "Chrome"], "metrics" => [1]}, + %{"dimensions" => ["Linux", "Firefox"], "metrics" => [2]} + ] + end + + test "breaks down all metrics by visit:utm_source with imported data", %{conn: conn, site: site} do + site_import = + insert(:site_import, + site: site, + start_date: ~D[2005-01-01], + end_date: Timex.today(), + source: :universal_analytics + ) + + populate_stats(site, site_import.id, [ + build(:pageview, utm_source: "SomeUTMSource", timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, utm_source: "SomeUTMSource-1", timestamp: ~N[2021-01-01 00:00:00]), + build(:imported_sources, + utm_source: "SomeUTMSource", + date: ~D[2021-01-01], + visitors: 2, + visits: 2, + pageviews: 2, + bounces: 1, + visit_duration: 120 + ), + build(:imported_sources, + utm_source: "SomeUTMSource-2", + date: ~D[2021-01-01], + visitors: 2, + visits: 2, + pageviews: 2, + bounces: 2, + visit_duration: 0 + ), + build(:imported_sources, + date: ~D[2021-01-01], + visitors: 10, + visits: 11, + pageviews: 50, + bounces: 0, + visit_duration: 1100 + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "visits", "pageviews", "bounce_rate", "visit_duration"], + "date_range" => "all", + "dimensions" => ["visit:utm_source"], + "include" => %{"imports" => true} + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["SomeUTMSource"], "metrics" => [3, 3, 3, 67.0, 40.0]}, + %{"dimensions" => ["SomeUTMSource-2"], "metrics" => [2, 2, 2, 100.0, 0.0]}, + %{"dimensions" => ["SomeUTMSource-1"], "metrics" => [1, 1, 1, 100.0, 0.0]} + ] + end + + # test "pageviews breakdown by event:page - imported data having pageviews=0 and visitors=n should be bypassed", + # %{conn: conn, site: site} do + # site_import = + # insert(:site_import, + # site: site, + # start_date: ~D[2005-01-01], + # end_date: Timex.today(), + # source: :universal_analytics + # ) + + # populate_stats(site, site_import.id, [ + # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]), + # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]), + # build(:pageview, + # pathname: "/plausible.io", + # timestamp: ~N[2021-01-01 00:00:00] + # ), + # build(:imported_pages, + # page: "/skip-me", + # date: ~D[2021-01-01], + # visitors: 1, + # pageviews: 0 + # ), + # build(:imported_pages, + # page: "/include-me", + # date: ~D[2021-01-01], + # visitors: 1, + # pageviews: 1 + # ) + # ]) + + # conn = + # get(conn, "/api/v1/stats/breakdown", %{ + # "site_id" => site.domain, + # "period" => "day", + # "date" => "2021-01-01", + # "property" => "event:page", + # "with_imported" => "true", + # "metrics" => "pageviews" + # }) + + # assert json_response(conn, 200) == %{ + # "results" => [ + # %{"page" => "/", "pageviews" => 2}, + # %{"page" => "/plausible.io", "pageviews" => 1}, + # %{"page" => "/include-me", "pageviews" => 1} + # ] + # } + # end + + # test "breakdown by event:page", %{conn: conn, site: site} do + # populate_stats(site, [ + # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]), + # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]), + # build(:pageview, + # pathname: "/plausible.io", + # timestamp: ~N[2021-01-01 00:00:00] + # ) + # ]) + + # conn = + # get(conn, "/api/v1/stats/breakdown", %{ + # "site_id" => site.domain, + # "period" => "day", + # "date" => "2021-01-01", + # "property" => "event:page" + # }) + + # assert json_response(conn, 200) == %{ + # "results" => [ + # %{"page" => "/", "visitors" => 2}, + # %{"page" => "/plausible.io", "visitors" => 1} + # ] + # } + # end + + # test "breakdown by event:page when there are no events in the second page", %{ + # conn: conn, + # site: site + # } do + # populate_stats(site, [ + # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]), + # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]), + # build(:pageview, + # pathname: "/plausible.io", + # timestamp: ~N[2021-01-01 00:00:00] + # ) + # ]) + + # conn = + # get(conn, "/api/v1/stats/breakdown", %{ + # "site_id" => site.domain, + # "period" => "day", + # "date" => "2021-01-01", + # "property" => "event:page", + # "metrics" => "visitors,bounce_rate", + # "page" => 2, + # "limit" => 2 + # }) + + # assert json_response(conn, 200) == %{"results" => []} + # end + + # test "attempting to breakdown by event:hostname returns an error", %{conn: conn, site: site} do + # conn = + # get(conn, "/api/v1/stats/breakdown", %{ + # "site_id" => site.domain, + # "period" => "day", + # "date" => "2021-01-01", + # "property" => "event:hostname", + # "with_imported" => "true" + # }) + + # assert %{ + # "error" => error + # } = json_response(conn, 400) + + # assert error =~ "Property 'event:hostname' is currently not supported for breakdowns." + # end + + describe "breakdown by visit:exit_page" do + setup %{site: site} do + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:pageview, + pathname: "/a", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: @user_id, + pathname: "/a", + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, + user_id: @user_id, + pathname: "/b", + timestamp: ~N[2021-01-01 00:35:00] + ), + build(:imported_exit_pages, + exit_page: "/b", + exits: 3, + visitors: 2, + pageviews: 5, + date: ~D[2021-01-01] + ) + ]) + end + + test "can query with visit:exit_page dimension", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visits"], + "date_range" => "all", + "dimensions" => ["visit:exit_page"], + "include" => %{"imports" => true} + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["/b"], "metrics" => [4]}, + %{"dimensions" => ["/a"], "metrics" => [1]} + ] + end + end + + describe "custom events" do + test "can breakdown by event:name", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Signup", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + timestamp: ~N[2021-01-01 00:25:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:name"] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Signup"], "metrics" => [2]}, + %{"dimensions" => ["pageview"], "metrics" => [1]} + ] + end + + test "can breakdown by event:name with visitors and events metrics", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/non-existing", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "404", + pathname: "/non-existing", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/non-existing", + timestamp: ~N[2021-01-01 00:00:01] + ), + build(:pageview, + pathname: "/non-existing", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:02] + ), + build(:event, + name: "404", + pathname: "/non-existing", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:02] + ), + build(:pageview, + pathname: "/non-existing", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:03] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "events"], + "date_range" => "all", + "dimensions" => ["event:name"] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["pageview"], "metrics" => [2, 4]}, + %{"dimensions" => ["404"], "metrics" => [1, 2]} + ] + end + + test "can breakdown by event:name while filtering for something", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Signup", + pathname: "/pageA", + browser: "Chrome", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + pathname: "/pageA", + browser: "Chrome", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + pathname: "/pageA", + browser: "Safari", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + pathname: "/pageB", + browser: "Chrome", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/pageA", + browser: "Chrome", + timestamp: ~N[2021-01-01 00:25:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:name"], + "filters" => [ + ["is", "event:page", ["/pageA"]], + ["is", "visit:browser", ["Chrome"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Signup"], "metrics" => [2]}, + %{"dimensions" => ["pageview"], "metrics" => [1]} + ] + end + + test "can breakdown by a visit:property when filtering by event:name", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + referrer_source: "Google", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + referrer_source: "Twitter", + timestamp: ~N[2021-01-01 00:25:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:source"], + "filters" => [ + ["is", "event:name", ["Signup"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Google"], "metrics" => [1]} + ] + end + + test "can breakdown by event:name when filtering by event:page", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/pageA", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/pageA", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + pathname: "/pageA", + name: "Signup", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/pageB", + referrer_source: "Twitter", + timestamp: ~N[2021-01-01 00:25:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:name"], + "filters" => [ + ["matches", "event:page", ["/pageA"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["pageview"], "metrics" => [2]}, + %{"dimensions" => ["Signup"], "metrics" => [1]} + ] + end + + test "can breakdown by event:page when filtering by event:name", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Signup", + pathname: "/pageA", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + pathname: "/pageA", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + pathname: "/pageB", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/pageB", + referrer_source: "Twitter", + timestamp: ~N[2021-01-01 00:25:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + ["is", "event:name", ["Signup"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["/pageA"], "metrics" => [2]}, + %{"dimensions" => ["/pageB"], "metrics" => [1]} + ] + end + + test "can filter event:page with a wildcard", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, pathname: "/en/page1"), + build(:pageview, pathname: "/en/page2"), + build(:pageview, pathname: "/en/page2"), + build(:pageview, pathname: "/pl/page1") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + ["matches", "event:page", ["/en/**"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["/en/page2"], "metrics" => [2]}, + %{"dimensions" => ["/en/page1"], "metrics" => [1]} + ] + end + + test "can filter event:hostname with a wildcard", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, hostname: "alice.example.com", pathname: "/a"), + build(:pageview, hostname: "anna.example.com", pathname: "/a"), + build(:pageview, hostname: "adam.example.com", pathname: "/a"), + build(:pageview, hostname: "bob.example.com", pathname: "/b") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + ["matches", "event:hostname", ["a*.example.com"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["/a"], "metrics" => [3]} + ] + end + + test "breakdown by custom event property", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Purchase", + "meta.key": ["package"], + "meta.value": ["business"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Purchase", + "meta.key": ["package"], + "meta.value": ["personal"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Purchase", + "meta.key": ["package"], + "meta.value": ["business"], + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:event, + name: "Some other event", + "meta.key": ["package"], + "meta.value": ["business"], + timestamp: ~N[2021-01-01 00:25:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:package"], + "filters" => [ + ["is", "event:name", ["Purchase"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["business"], "metrics" => [2]}, + %{"dimensions" => ["personal"], "metrics" => [1]} + ] + end + + test "breakdown by custom event property, with pageviews metric", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + "meta.key": ["package"], + "meta.value": ["business"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + "meta.key": ["package"], + "meta.value": ["personal"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + "meta.key": ["package"], + "meta.value": ["business"], + timestamp: ~N[2021-01-01 00:25:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["pageviews"], + "date_range" => "all", + "dimensions" => ["event:props:package"], + "filters" => [] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["business"], "metrics" => [2]}, + %{"dimensions" => ["personal"], "metrics" => [1]} + ] + end + + test "breakdown by custom event property, with (none)", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["16"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["16"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["16"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Purchase", + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["14"], + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["14"], + timestamp: ~N[2021-01-01 00:26:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:cost"], + "filters" => [["is", "event:name", ["Purchase"]]] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["16"], "metrics" => [3]}, + %{"dimensions" => ["14"], "metrics" => [2]}, + %{"dimensions" => ["(none)"], "metrics" => [1]} + ] + end + + # test "breakdown by custom event property, limited", %{conn: conn, site: site} do + # populate_stats(site, [ + # build(:event, + # name: "Purchase", + # "meta.key": ["cost"], + # "meta.value": ["16"], + # timestamp: ~N[2021-01-01 00:00:00] + # ), + # build(:event, + # name: "Purchase", + # "meta.key": ["cost"], + # "meta.value": ["18"], + # timestamp: ~N[2021-01-01 00:25:00] + # ), + # build(:event, + # name: "Purchase", + # "meta.key": ["cost"], + # "meta.value": ["14"], + # timestamp: ~N[2021-01-01 00:25:00] + # ), + # build(:event, + # name: "Purchase", + # "meta.key": ["cost"], + # "meta.value": ["14"], + # timestamp: ~N[2021-01-01 00:26:00] + # ) + # ]) + + # conn = + # get(conn, "/api/v1/stats/breakdown", %{ + # "site_id" => site.domain, + # "period" => "day", + # "date" => "2021-01-01", + # "property" => "event:props:cost", + # "filters" => "event:name==Purchase", + # "limit" => 2 + # }) + + # assert json_response(conn, 200) == %{ + # "results" => [ + # %{"cost" => "14", "visitors" => 2}, + # %{"cost" => "16", "visitors" => 1} + # ] + # } + # end + + # test "breakdown by custom event property, paginated", %{conn: conn, site: site} do + # populate_stats(site, [ + # build(:event, + # name: "Purchase", + # "meta.key": ["cost"], + # "meta.value": ["16"], + # timestamp: ~N[2021-01-01 00:00:00] + # ), + # build(:event, + # name: "Purchase", + # "meta.key": ["cost"], + # "meta.value": ["16"], + # timestamp: ~N[2021-01-01 00:00:00] + # ), + # build(:event, + # name: "Purchase", + # "meta.key": ["cost"], + # "meta.value": ["18"], + # timestamp: ~N[2021-01-01 00:25:00] + # ), + # build(:event, + # name: "Purchase", + # "meta.key": ["cost"], + # "meta.value": ["14"], + # timestamp: ~N[2021-01-01 00:25:00] + # ), + # build(:event, + # name: "Purchase", + # "meta.key": ["cost"], + # "meta.value": ["14"], + # timestamp: ~N[2021-01-01 00:26:00] + # ) + # ]) + + # conn = + # get(conn, "/api/v1/stats/breakdown", %{ + # "site_id" => site.domain, + # "period" => "day", + # "date" => "2021-01-01", + # "property" => "event:props:cost", + # "filters" => "event:name==Purchase", + # "limit" => 2, + # "page" => 2 + # }) + + # assert json_response(conn, 200) == %{ + # "results" => [ + # %{"cost" => "18", "visitors" => 1} + # ] + # } + # end + end + + # describe "breakdown by event:goal" do + # test "returns custom event goals and pageview goals", %{conn: conn, site: site} do + # insert(:goal, %{site: site, event_name: "Purchase"}) + # insert(:goal, %{site: site, page_path: "/test"}) + + # populate_stats(site, [ + # build(:pageview, + # timestamp: ~N[2021-01-01 00:00:01], + # pathname: "/test" + # ), + # build(:event, + # name: "Purchase", + # timestamp: ~N[2021-01-01 00:00:03] + # ), + # build(:event, + # name: "Purchase", + # timestamp: ~N[2021-01-01 00:00:03] + # ) + # ]) + + # conn = + # get(conn, "/api/v1/stats/breakdown", %{ + # "site_id" => site.domain, + # "period" => "day", + # "date" => "2021-01-01", + # "property" => "event:goal" + # }) + + # assert [ + # %{"goal" => "Purchase", "visitors" => 2}, + # %{"goal" => "Visit /test", "visitors" => 1} + # ] = json_response(conn, 200)["results"] + # end + + # test "returns pageview goals containing wildcards", %{conn: conn, site: site} do + # insert(:goal, %{site: site, page_path: "/**/post"}) + # insert(:goal, %{site: site, page_path: "/blog**"}) + + # populate_stats(site, [ + # build(:pageview, pathname: "/blog", user_id: @user_id), + # build(:pageview, pathname: "/blog/post-1", user_id: @user_id), + # build(:pageview, pathname: "/blog/post-2", user_id: @user_id), + # build(:pageview, pathname: "/blog/something/post"), + # build(:pageview, pathname: "/different/page/post") + # ]) + + # conn = + # get(conn, "/api/v1/stats/breakdown", %{ + # "site_id" => site.domain, + # "metrics" => "visitors,pageviews", + # "property" => "event:goal" + # }) + + # assert [ + # %{"goal" => "Visit /**/post", "visitors" => 2, "pageviews" => 2}, + # %{"goal" => "Visit /blog**", "visitors" => 2, "pageviews" => 4} + # ] = json_response(conn, 200)["results"] + # end + + # test "does not return goals that are not configured for the site", %{conn: conn, site: site} do + # populate_stats(site, [ + # build(:pageview, pathname: "/register"), + # build(:event, name: "Signup") + # ]) + + # conn = + # get(conn, "/api/v1/stats/breakdown", %{ + # "site_id" => site.domain, + # "metrics" => "visitors,pageviews", + # "property" => "event:goal" + # }) + + # assert [] = json_response(conn, 200)["results"] + # end + # end + + # describe "filtering" do + test "event:goal filter returns 400 when goal not configured", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:browser"], + "filters" => [ + ["is", "event:goal", ["Register"]] + ] + }) + + assert %{"error" => msg} = json_response(conn, 400) + assert msg =~ "The goal `Register` is not configured for this site. Find out how" + end + + test "validates that filters are valid", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:browser"], + "filters" => [ + ["is", "badproperty", ["bar"]] + ] + }) + + assert %{"error" => msg} = json_response(conn, 400) + assert msg =~ "Invalid filter" + end + + test "event:page filter for breakdown by session props", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/ignore", + browser: "Chrome", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/plausible.io", + browser: "Chrome", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/plausible.io", + browser: "Chrome", + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, + browser: "Safari", + pathname: "/plausible.io", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:browser"], + "filters" => [ + ["is", "event:page", ["/plausible.io"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Chrome"], "metrics" => [2]}, + %{"dimensions" => ["Safari"], "metrics" => [1]} + ] + end + + test "event:page filter shows sources of sessions that have visited that page", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + pathname: "/", + referrer_source: "Twitter", + utm_medium: "Twitter", + utm_source: "Twitter", + utm_campaign: "Twitter", + user_id: @user_id + ), + build(:pageview, + pathname: "/plausible.io", + user_id: @user_id + ), + build(:pageview, + pathname: "/plausible.io", + referrer_source: "Google", + utm_medium: "Google", + utm_source: "Google", + utm_campaign: "Google" + ), + build(:pageview, + pathname: "/plausible.io", + referrer_source: "Google", + utm_medium: "Google", + utm_source: "Google", + utm_campaign: "Google" + ) + ]) + + for dimension <- [ + "visit:source", + "visit:utm_medium", + "visit:utm_source", + "visit:utm_campaign" + ] do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => [dimension], + "filters" => [ + ["is", "event:page", ["/plausible.io"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Google"], "metrics" => [2]}, + %{"dimensions" => ["Twitter"], "metrics" => [1]} + ] + end + end + + # test "top sources for a custom goal and filtered by hostname", %{conn: conn, site: site} do + # populate_stats(site, [ + # build(:pageview, + # hostname: "blog.example.com", + # referrer_source: "Facebook", + # user_id: @user_id + # ), + # build(:pageview, + # hostname: "app.example.com", + # pathname: "/register", + # user_id: @user_id + # ), + # build(:event, + # name: "Signup", + # hostname: "app.example.com", + # pathname: "/register", + # user_id: @user_id + # ) + # ]) + + # conn = + # post(conn, "/api/v2/query", %{ + # "site_id" => site.domain, + # "metrics" => ["visitors"], + # "date_range" => "all", + # "dimensions" => ["visit:source"], + # "filters" => [ + # ["is", "event:hostname", ["app.example.com"]] + # ] + # }) + + # %{"results" => results} = json_response(conn, 200) + + # assert results == [] + # end + + # test "top sources for a custom goal and filtered by hostname (2)", %{conn: conn, site: site} do + # populate_stats(site, [ + # build(:pageview, + # hostname: "app.example.com", + # referrer_source: "Facebook", + # pathname: "/register", + # user_id: @user_id + # ), + # build(:event, + # name: "Signup", + # hostname: "app.example.com", + # pathname: "/register", + # user_id: @user_id + # ) + # ]) + + # conn = + # get(conn, "/api/v1/stats/breakdown", %{ + # "site_id" => site.domain, + # "period" => "day", + # "property" => "visit:source", + # "filters" => "event:hostname==app.example.com" + # }) + + # assert json_response(conn, 200) == %{ + # "results" => [%{"source" => "Facebook", "visitors" => 1}] + # } + # end + + # test "event:page filter is interpreted as entry_page filter only for bounce_rate", %{ + # conn: conn, + # site: site + # } do + # populate_stats(site, [ + # build(:pageview, + # browser: "Chrome", + # user_id: @user_id, + # pathname: "/ignore", + # timestamp: ~N[2021-01-01 00:00:00] + # ), + # build(:pageview, + # browser: "Chrome", + # user_id: @user_id, + # pathname: "/plausible.io", + # timestamp: ~N[2021-01-01 00:01:00] + # ), + # build(:pageview, + # browser: "Chrome", + # user_id: 456, + # pathname: "/important-page", + # timestamp: ~N[2021-01-01 00:00:00] + # ), + # build(:pageview, + # browser: "Chrome", + # user_id: 456, + # pathname: "/", + # timestamp: ~N[2021-01-01 00:01:00] + # ), + # build(:pageview, + # browser: "Chrome", + # pathname: "/plausible.io", + # timestamp: ~N[2021-01-01 00:01:00] + # ) + # ]) + + # conn = + # get(conn, "/api/v1/stats/breakdown", %{ + # "site_id" => site.domain, + # "period" => "day", + # "date" => "2021-01-01", + # "metrics" => "visitors,bounce_rate", + # "property" => "visit:browser", + # "filters" => "event:page == /plausible.io|/important-page" + # }) + + # assert json_response(conn, 200) == %{ + # "results" => [ + # %{ + # "browser" => "Chrome", + # "bounce_rate" => 50, + # "visitors" => 3 + # } + # ] + # } + # end + + test "event:goal pageview filter for breakdown by visit source", %{conn: conn, site: site} do + insert(:goal, %{site: site, page_path: "/plausible.io"}) + + populate_stats(site, [ + build(:pageview, + referrer_source: "Bing", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + referrer_source: "Google", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/plausible.io", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:source"], + "filters" => [ + ["is", "event:goal", ["Visit /plausible.io"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Google"], "metrics" => [1]} + ] + end + + test "event:goal custom event filter for breakdown by visit source", %{conn: conn, site: site} do + insert(:goal, %{site: site, event_name: "Register"}) + + populate_stats(site, [ + build(:pageview, + referrer_source: "Bing", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + referrer_source: "Google", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Register", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:source"], + "filters" => [ + ["is", "event:goal", ["Register"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Google"], "metrics" => [1]} + ] + end + + test "wildcard pageview goal filter for breakdown by event:page", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/en/register"), + build(:pageview, pathname: "/en/register", user_id: @user_id), + build(:pageview, pathname: "/en/register", user_id: @user_id), + build(:pageview, pathname: "/123/it/register"), + build(:pageview, pathname: "/should-not-appear") + ]) + + insert(:goal, %{site: site, page_path: "/**register"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "pageviews"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + ["matches", "event:goal", ["Visit /**register"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["/en/register"], "metrics" => [2, 3]}, + %{"dimensions" => ["/123/it/register"], "metrics" => [1, 1]} + ] + end + + test "mixed multi-goal filter for breakdown by visit:country", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, country_code: "EE", pathname: "/en/register"), + build(:event, country_code: "EE", name: "Signup", pathname: "/en/register"), + build(:pageview, country_code: "US", pathname: "/123/it/register"), + build(:pageview, country_code: "US", pathname: "/different") + ]) + + insert(:goal, %{site: site, page_path: "/**register"}) + insert(:goal, %{site: site, event_name: "Signup"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "pageviews", "events"], + "date_range" => "all", + "dimensions" => ["visit:country"], + "filters" => [ + ["matches", "event:goal", ["Signup", "Visit /**register"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["EE"], "metrics" => [2, 1, 2]}, + %{"dimensions" => ["US"], "metrics" => [1, 1, 1]} + ] + end + + test "event:goal custom event filter for breakdown by event page", %{conn: conn, site: site} do + insert(:goal, %{site: site, event_name: "Register"}) + + populate_stats(site, [ + build(:event, + pathname: "/en/register", + name: "Register" + ), + build(:event, + pathname: "/en/register", + name: "Register" + ), + build(:event, + pathname: "/it/register", + name: "Register" + ) + ]) + + conn = + get(conn, "/api/v1/stats/breakdown", %{ + "site_id" => site.domain, + "period" => "day", + "property" => "event:page", + "filters" => "event:goal == Register" + }) + + assert json_response(conn, 200) == %{ + "results" => [ + %{"page" => "/en/register", "visitors" => 2}, + %{"page" => "/it/register", "visitors" => 1} + ] + } + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + ["is", "event:goal", ["Register"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["/en/register"], "metrics" => [2]}, + %{"dimensions" => ["/it/register"], "metrics" => [1]} + ] + end + + test "IN filter for event:page", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/ignore", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/plausible.io", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/plausible.io", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/important-page", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + ["is", "event:page", ["/plausible.io", "/important-page"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["/plausible.io"], "metrics" => [2]}, + %{"dimensions" => ["/important-page"], "metrics" => [1]} + ] + end + + test "IN filter for visit:browser", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/ignore", + browser: "Firefox", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/plausible.io", + browser: "Chrome", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/plausible.io", + browser: "Safari", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/important-page", + browser: "Safari", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:page"], + "filters" => [ + ["is", "visit:browser", ["Chrome", "Safari"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["/plausible.io"], "metrics" => [2]}, + %{"dimensions" => ["/important-page"], "metrics" => [1]} + ] + end + + # test "IN filter for visit:entry_page", %{conn: conn, site: site} do + # populate_stats(site, [ + # build(:pageview, + # pathname: "/ignore", + # timestamp: ~N[2021-01-01 00:00:00] + # ), + # build(:pageview, + # pathname: "/plausible.io", + # timestamp: ~N[2021-01-01 00:00:00] + # ), + # build(:pageview, + # pathname: "/plausible.io", + # timestamp: ~N[2021-01-01 00:00:00] + # ), + # build(:pageview, + # pathname: "/important-page", + # timestamp: ~N[2021-01-01 00:00:00] + # ) + # ]) + + # conn = + # post(conn, "/api/v2/query", %{ + # "site_id" => site.domain, + # "metrics" => ["bounce_rate"], + # "date_range" => "all", + # "dimensions" => ["event:page"], + # "filters" => [ + # ["is", "event:page", ["/plausible.io", "/important-page"]] + # ] + # }) + + # %{"results" => results} = json_response(conn, 200) + + # assert results == [ + # %{"dimensions" => ["plausible.io"], "metrics" => [100]}, + # %{"dimensions" => ["/important-page"], "metrics" => [100]}, + # ] + # end + + test "IN filter for event:name", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Signup", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Login", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Irrelevant", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:name"], + "filters" => [ + ["is", "event:name", ["Signup", "Login"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Signup"], "metrics" => [2]}, + %{"dimensions" => ["Login"], "metrics" => [1]} + ] + end + + test "IN filter for event:props:*", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + browser: "Chrome", + "meta.key": ["browser"], + "meta.value": ["Chrome"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Chrome", + "meta.key": ["browser"], + "meta.value": ["Chrome"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Safari", + "meta.key": ["browser"], + "meta.value": ["Safari"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Firefox", + "meta.key": ["browser"], + "meta.value": ["Firefox"], + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:browser"], + "filters" => [ + ["is", "event:props:browser", ["Chrome", "Safari"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Chrome"], "metrics" => [2]}, + %{"dimensions" => ["Safari"], "metrics" => [1]} + ] + end + + test "Multiple event:props:* filters", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + browser: "Chrome", + "meta.key": ["browser"], + "meta.value": ["Chrome"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Chrome", + "meta.key": ["browser", "prop"], + "meta.value": ["Chrome", "xyz"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Safari", + "meta.key": ["browser", "prop"], + "meta.value": ["Safari", "target_value"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Firefox", + "meta.key": ["browser", "prop"], + "meta.value": ["Firefox", "target_value"], + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:browser"], + "filters" => [ + ["is", "event:props:browser", ["Chrome", "Safari"]], + ["is", "event:props:prop", ["target_value"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Safari"], "metrics" => [1]} + ] + end + + test "IN filter for event:props:* including (none) value", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + browser: "Chrome", + "meta.key": ["browser"], + "meta.value": ["Chrome"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Chrome", + "meta.key": ["browser"], + "meta.value": ["Chrome"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Safari", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Firefox", + "meta.key": ["browser"], + "meta.value": ["Firefox"], + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:browser"], + "filters" => [["is", "event:props:browser", ["Chrome", "(none)"]]] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Chrome"], "metrics" => [2]}, + %{"dimensions" => ["Safari"], "metrics" => [1]} + ] + end + + test "can use a is_not filter", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, browser: "Chrome"), + build(:pageview, browser: "Safari"), + build(:pageview, browser: "Safari"), + build(:pageview, browser: "Edge") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:browser"], + "filters" => [ + ["is_not", "visit:browser", ["Chrome"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Safari"], "metrics" => [2]}, + %{"dimensions" => ["Edge"], "metrics" => [1]} + ] + end + + # describe "metrics" do + # test "returns conversion_rate in an event:goal breakdown", %{conn: conn, site: site} do + # populate_stats(site, [ + # build(:event, name: "Signup", user_id: 1), + # build(:event, name: "Signup", user_id: 1), + # build(:pageview, pathname: "/blog"), + # build(:pageview, pathname: "/blog/post"), + # build(:pageview) + # ]) + + # insert(:goal, %{site: site, event_name: "Signup"}) + # insert(:goal, %{site: site, page_path: "/blog**"}) + + # conn = + # get(conn, "/api/v1/stats/breakdown", %{ + # "site_id" => site.domain, + # "period" => "day", + # "property" => "event:goal", + # "metrics" => "visitors,events,conversion_rate" + # }) + + # assert json_response(conn, 200) == %{ + # "results" => [ + # %{ + # "goal" => "Visit /blog**", + # "visitors" => 2, + # "events" => 2, + # "conversion_rate" => 50 + # }, + # %{ + # "goal" => "Signup", + # "visitors" => 1, + # "events" => 2, + # "conversion_rate" => 25 + # } + # ] + # } + # end + + # test "returns conversion_rate alone in an event:goal breakdown", %{conn: conn, site: site} do + # populate_stats(site, [ + # build(:event, name: "Signup", user_id: 1), + # build(:pageview) + # ]) + + # insert(:goal, %{site: site, event_name: "Signup"}) + + # conn = + # get(conn, "/api/v1/stats/breakdown", %{ + # "site_id" => site.domain, + # "period" => "day", + # "property" => "event:goal", + # "metrics" => "conversion_rate" + # }) + + # assert json_response(conn, 200) == %{ + # "results" => [ + # %{ + # "goal" => "Signup", + # "conversion_rate" => 50 + # } + # ] + # } + # end + + # test "returns conversion_rate in a goal filtered custom prop breakdown", %{ + # conn: conn, + # site: site + # } do + # populate_stats(site, [ + # build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]), + # build(:pageview, pathname: "/blog/2", "meta.key": ["author"], "meta.value": ["Uku"]), + # build(:pageview, pathname: "/blog/3", "meta.key": ["author"], "meta.value": ["Uku"]), + # build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Marko"]), + # build(:pageview, + # pathname: "/blog/2", + # "meta.key": ["author"], + # "meta.value": ["Marko"], + # user_id: 1 + # ), + # build(:pageview, + # pathname: "/blog/3", + # "meta.key": ["author"], + # "meta.value": ["Marko"], + # user_id: 1 + # ), + # build(:pageview, pathname: "/blog"), + # build(:pageview, "meta.key": ["author"], "meta.value": ["Marko"]), + # build(:pageview) + # ]) + + # insert(:goal, %{site: site, page_path: "/blog**"}) + + # conn = + # get(conn, "/api/v1/stats/breakdown", %{ + # "site_id" => site.domain, + # "period" => "day", + # "property" => "event:props:author", + # "filters" => "event:goal==Visit /blog**", + # "metrics" => "visitors,events,conversion_rate" + # }) + + # assert json_response(conn, 200) == %{ + # "results" => [ + # %{ + # "author" => "Uku", + # "visitors" => 3, + # "events" => 3, + # "conversion_rate" => 37.5 + # }, + # %{ + # "author" => "Marko", + # "visitors" => 2, + # "events" => 3, + # "conversion_rate" => 25 + # }, + # %{ + # "author" => "(none)", + # "visitors" => 1, + # "events" => 1, + # "conversion_rate" => 12.5 + # } + # ] + # } + # end + + test "returns conversion_rate alone in a goal filtered custom prop breakdown", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]), + build(:pageview) + ]) + + insert(:goal, %{site: site, page_path: "/blog**"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["conversion_rate"], + "date_range" => "all", + "dimensions" => ["event:props:author"], + "filters" => [["matches", "event:goal", ["Visit /blog**"]]] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Uku"], "metrics" => [50]} + ] + end + + # test "returns conversion_rate in a goal filtered event:page breakdown", %{ + # conn: conn, + # site: site + # } do + # populate_stats(site, [ + # build(:event, pathname: "/en/register", name: "pageview"), + # build(:event, pathname: "/en/register", name: "Signup"), + # build(:event, pathname: "/en/register", name: "Signup"), + # build(:event, pathname: "/it/register", name: "Signup", user_id: 1), + # build(:event, pathname: "/it/register", name: "Signup", user_id: 1), + # build(:event, pathname: "/it/register", name: "pageview") + # ]) + + # insert(:goal, %{site: site, event_name: "Signup"}) + + # conn = + # get(conn, "/api/v1/stats/breakdown", %{ + # "site_id" => site.domain, + # "period" => "day", + # "property" => "event:page", + # "filters" => "event:goal==Signup", + # "metrics" => "visitors,events,conversion_rate" + # }) + + # assert json_response(conn, 200) == %{ + # "results" => [ + # %{ + # "page" => "/en/register", + # "visitors" => 2, + # "events" => 2, + # "conversion_rate" => 66.7 + # }, + # %{ + # "page" => "/it/register", + # "visitors" => 1, + # "events" => 2, + # "conversion_rate" => 50 + # } + # ] + # } + # end + + # test "returns conversion_rate alone in a goal filtered event:page breakdown", %{ + # conn: conn, + # site: site + # } do + # populate_stats(site, [ + # build(:event, pathname: "/en/register", name: "pageview"), + # build(:event, pathname: "/en/register", name: "Signup") + # ]) + + # insert(:goal, %{site: site, event_name: "Signup"}) + + # conn = + # get(conn, "/api/v1/stats/breakdown", %{ + # "site_id" => site.domain, + # "period" => "day", + # "property" => "event:page", + # "filters" => "event:goal==Signup", + # "metrics" => "conversion_rate" + # }) + + # assert json_response(conn, 200) == %{ + # "results" => [ + # %{ + # "page" => "/en/register", + # "conversion_rate" => 50 + # } + # ] + # } + # end + + # test "returns conversion_rate in a multi-goal filtered visit:screen_size breakdown", %{ + # conn: conn, + # site: site + # } do + # populate_stats(site, [ + # build(:event, screen_size: "Mobile", name: "pageview"), + # build(:event, screen_size: "Mobile", name: "AddToCart"), + # build(:event, screen_size: "Mobile", name: "AddToCart"), + # build(:event, screen_size: "Desktop", name: "AddToCart", user_id: 1), + # build(:event, screen_size: "Desktop", name: "Purchase", user_id: 1), + # build(:event, screen_size: "Desktop", name: "pageview") + # ]) + + # # Make sure that revenue goals are treated the same + # # way as regular custom event goals + # insert(:goal, %{site: site, event_name: "Purchase", currency: :EUR}) + # insert(:goal, %{site: site, event_name: "AddToCart"}) + + # conn = + # post(conn, "/api/v2/query", %{ + # "site_id" => site.domain, + # "metrics" => ["visitors", "events", "conversion_rate"], + # "date_range" => "all", + # "dimensions" => ["visit:device"], + # "filters" => [["is", "event:goal", ["AddToCart", "Purchase"]]] + # }) + + # %{"results" => results} = json_response(conn, 200) + + # assert results == [ + # %{"dimensions" => ["Mobile"], "metrics" => [2, 2, 66.7]}, + # %{"dimensions" => ["Desktop"], "metrics" => [1, 2, 50]}, + # ] + # end + + test "returns conversion_rate alone in a goal filtered visit:screen_size breakdown", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, screen_size: "Mobile", name: "pageview"), + build(:event, screen_size: "Mobile", name: "AddToCart") + ]) + + insert(:goal, %{site: site, event_name: "AddToCart"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["conversion_rate"], + "date_range" => "all", + "dimensions" => ["visit:device"], + "filters" => [["is", "event:goal", ["AddToCart"]]] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Mobile"], "metrics" => [50]} + ] + end + + # test "returns conversion_rate for a browser_version breakdown with pagination limit", %{ + # site: site, + # conn: conn + # } do + # populate_stats(site, [ + # build(:pageview, browser: "Firefox", browser_version: "110"), + # build(:pageview, browser: "Firefox", browser_version: "110"), + # build(:pageview, browser: "Chrome", browser_version: "110"), + # build(:pageview, browser: "Chrome", browser_version: "110"), + # build(:pageview, browser: "Avast Secure Browser", browser_version: "110"), + # build(:pageview, browser: "Avast Secure Browser", browser_version: "110"), + # build(:event, name: "Signup", browser: "Edge", browser_version: "110") + # ]) + + # insert(:goal, site: site, event_name: "Signup") + + # conn = + # get(conn, "/api/v1/stats/breakdown", %{ + # "site_id" => site.domain, + # "period" => "day", + # "property" => "visit:browser_version", + # "filters" => "event:goal==Signup", + # "metrics" => "visitors,conversion_rate", + # "page" => 1, + # "limit" => 1 + # }) + + # assert json_response(conn, 200) == %{ + # "results" => [ + # %{ + # "browser" => "Edge", + # "browser_version" => "110", + # "visitors" => 1, + # "conversion_rate" => 100.0 + # } + # ] + # } + # end + + test "all metrics for breakdown by visit prop", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + user_id: 1, + referrer_source: "Google", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "signup", + user_id: 1, + referrer_source: "Google", + timestamp: ~N[2021-01-01 00:05:00] + ), + build(:pageview, + user_id: 1, + referrer_source: "Google", + timestamp: ~N[2021-01-01 00:10:00] + ), + build(:pageview, + referrer_source: "Google", + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, + referrer_source: "Twitter", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => [ + "visitors", + "visits", + "pageviews", + "events", + "bounce_rate", + "visit_duration" + ], + "date_range" => "all", + "dimensions" => ["visit:source"], + "filters" => [] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Google"], "metrics" => [2, 2, 3, 4, 50, 300]}, + %{"dimensions" => ["Twitter"], "metrics" => [1, 1, 1, 1, 100, 0]} + ] + end + + test "metrics=bounce_rate does not add visits to the response", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + user_id: 1, + pathname: "/entry-page-1", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: 1, + pathname: "/some-page", + timestamp: ~N[2021-01-01 00:10:00] + ), + build(:pageview, + user_id: 2, + pathname: "/entry-page-2", + referrer_source: "Google", + timestamp: ~N[2021-01-01 00:05:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["bounce_rate"], + "date_range" => "all", + "dimensions" => ["visit:entry_page"], + "order_by" => [["visit:entry_page", "asc"]] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["/entry-page-1"], "metrics" => [0]}, + %{"dimensions" => ["/entry-page-2"], "metrics" => [100]} + ] + end + + test "filter by custom event property", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Purchase", + "meta.key": ["package"], + "meta.value": ["business"], + browser: "Chrome", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Purchase", + "meta.key": ["package"], + "meta.value": ["business"], + browser: "Safari", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Purchase", + "meta.key": ["package"], + "meta.value": ["business"], + browser: "Safari", + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:event, + name: "Purchase", + "meta.key": ["package"], + "meta.value": ["personal"], + browser: "IE", + timestamp: ~N[2021-01-01 00:25:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visit:browser"], + "filters" => [ + ["is", "event:name", ["Purchase"]], + ["is", "event:props:package", ["business"]] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Safari"], "metrics" => [2]}, + %{"dimensions" => ["Chrome"], "metrics" => [1]} + ] + end + + # test "all metrics for breakdown by event prop", %{conn: conn, site: site} do + # populate_stats(site, [ + # build(:pageview, + # user_id: 1, + # pathname: "/", + # timestamp: ~N[2021-01-01 00:00:00] + # ), + # build(:pageview, + # user_id: 1, + # pathname: "/plausible.io", + # timestamp: ~N[2021-01-01 00:10:00] + # ), + # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]), + # build(:pageview, + # pathname: "/plausible.io", + # timestamp: ~N[2021-01-01 00:00:00] + # ) + # ]) + + # conn = + # post(conn, "/api/v2/query", %{ + # "site_id" => site.domain, + # "metrics" => [ + # "visitors", + # "visits", + # "pageviews", + # "events", + # "bounce_rate", + # "visit_duration" + # ], + # "date_range" => "all", + # "dimensions" => ["event:page"], + # }) + + # %{"results" => results} = json_response(conn, 200) + + # assert results == [ + # %{"dimensions" => ["/"], "metrics" => [2, 2, 3, 4, 50, 300]}, + # %{"dimensions" => ["/plausible.io"], "metrics" => [1, 1, 1, 1, 100, 0]} + # ] + # end + # end + + describe "imported data" do + test "returns screen sizes breakdown when filtering by screen size", %{conn: conn, site: site} do + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:pageview, + timestamp: ~N[2021-01-01 00:00:01], + screen_size: "Mobile" + ), + build(:imported_devices, + device: "Mobile", + visitors: 3, + pageviews: 5, + date: ~D[2021-01-01] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "pageviews"], + "date_range" => "all", + "dimensions" => ["visit:device"], + "filters" => [ + ["is", "visit:device", ["Mobile"]] + ], + "include" => %{"imports" => true} + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [%{"dimensions" => ["Mobile"], "metrics" => [4, 6]}] + end + + # test "returns custom event goals and pageview goals", %{conn: conn, site: site} do + # insert(:goal, site: site, event_name: "Purchase") + # insert(:goal, site: site, page_path: "/test") + + # site_import = insert(:site_import, site: site) + + # populate_stats(site, site_import.id, [ + # build(:pageview, + # timestamp: ~N[2021-01-01 00:00:01], + # pathname: "/test" + # ), + # build(:event, + # name: "Purchase", + # timestamp: ~N[2021-01-01 00:00:03] + # ), + # build(:event, + # name: "Purchase", + # timestamp: ~N[2021-01-01 00:00:03] + # ), + # build(:imported_custom_events, + # name: "Purchase", + # visitors: 3, + # events: 5, + # date: ~D[2021-01-01] + # ), + # build(:imported_pages, + # page: "/test", + # visitors: 2, + # pageviews: 2, + # date: ~D[2021-01-01] + # ), + # build(:imported_visitors, visitors: 5, date: ~D[2021-01-01]) + # ]) + + # conn = + # get(conn, "/api/v1/stats/breakdown", %{ + # "site_id" => site.domain, + # "period" => "day", + # "date" => "2021-01-01", + # "property" => "event:goal", + # "metrics" => "visitors,events,pageviews,conversion_rate", + # "with_imported" => "true" + # }) + + # assert [ + # %{ + # "goal" => "Purchase", + # "visitors" => 5, + # "events" => 7, + # "pageviews" => 0, + # "conversion_rate" => 62.5 + # }, + # %{ + # "goal" => "Visit /test", + # "visitors" => 3, + # "events" => 3, + # "pageviews" => 3, + # "conversion_rate" => 37.5 + # } + # ] = json_response(conn, 200)["results"] + # end + + test "pageviews are returned as events for breakdown reports other than custom events", %{ + conn: conn, + site: site + } do + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:imported_browsers, browser: "Chrome", pageviews: 1, date: ~D[2021-01-01]), + build(:imported_devices, device: "Desktop", pageviews: 1, date: ~D[2021-01-01]), + build(:imported_entry_pages, entry_page: "/test", pageviews: 1, date: ~D[2021-01-01]), + build(:imported_exit_pages, exit_page: "/test", pageviews: 1, date: ~D[2021-01-01]), + build(:imported_locations, country: "EE", pageviews: 1, date: ~D[2021-01-01]), + build(:imported_operating_systems, + operating_system: "Mac", + pageviews: 1, + date: ~D[2021-01-01] + ), + build(:imported_pages, page: "/test", pageviews: 1, date: ~D[2021-01-01]), + build(:imported_sources, source: "Google", pageviews: 1, date: ~D[2021-01-01]) + ]) + + breakdown_and_first = fn dimension -> + conn + |> post("/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["events"], + "date_range" => ["2021-01-01", "2021-01-01"], + "dimensions" => [dimension], + "include" => %{"imports" => true} + }) + |> json_response(200) + |> Map.get("results") + |> List.first() + end + + assert %{"dimensions" => ["Chrome"], "metrics" => [1]} = + breakdown_and_first.("visit:browser") + + assert %{"dimensions" => ["Desktop"], "metrics" => [1]} = + breakdown_and_first.("visit:device") + + # assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("visit:entry_page") + # assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("visit:exit_page") + assert %{"dimensions" => ["EE"], "metrics" => [1]} = breakdown_and_first.("visit:country") + assert %{"dimensions" => ["Mac"], "metrics" => [1]} = breakdown_and_first.("visit:os") + assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("event:page") + + assert %{"dimensions" => ["Google"], "metrics" => [1]} = + breakdown_and_first.("visit:source") + end + + for goal_name <- Plausible.Imported.goals_with_url() do + test "returns url breakdown for #{goal_name} goal", %{conn: conn, site: site} do + insert(:goal, event_name: unquote(goal_name), site: site) + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:event, + name: unquote(goal_name), + "meta.key": ["url"], + "meta.value": ["https://one.com"] + ), + build(:imported_custom_events, + name: unquote(goal_name), + visitors: 2, + events: 5, + link_url: "https://one.com" + ), + build(:imported_custom_events, + name: unquote(goal_name), + visitors: 5, + events: 10, + link_url: "https://two.com" + ), + build(:imported_custom_events, + name: "some goal", + visitors: 5, + events: 10 + ), + build(:imported_visitors, visitors: 9) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "events"], + # "metrics" => ["visitors", "events", "conversion_rate"], + "date_range" => "all", + "dimensions" => ["event:props:url"], + "filters" => [ + ["is", "event:goal", [unquote(goal_name)]] + ], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["https://two.com"], "metrics" => [5, 10]}, + %{"dimensions" => ["https://one.com"], "metrics" => [3, 6]} + ] + + refute json_response(conn, 200)["meta"]["warning"] + end + end + + for goal_name <- Plausible.Imported.goals_with_path() do + test "returns path breakdown for #{goal_name} goal", %{conn: conn, site: site} do + insert(:goal, event_name: unquote(goal_name), site: site) + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:event, + name: unquote(goal_name), + "meta.key": ["path"], + "meta.value": ["/one"] + ), + build(:imported_custom_events, + name: unquote(goal_name), + visitors: 2, + events: 5, + path: "/one" + ), + build(:imported_custom_events, + name: unquote(goal_name), + visitors: 5, + events: 10, + path: "/two" + ), + build(:imported_custom_events, + name: "some goal", + visitors: 5, + events: 10 + ), + build(:imported_visitors, visitors: 9) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors", "events"], + # "metrics" => ["visitors", "events", "conversion_rate"], + "date_range" => "all", + "dimensions" => ["event:props:path"], + "filters" => [ + ["is", "event:goal", [unquote(goal_name)]] + ], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["/two"], "metrics" => [5, 10]}, + %{"dimensions" => ["/one"], "metrics" => [3, 6]} + ] + + refute json_response(conn, 200)["meta"]["warning"] + end + end + + test "adds a warning when query params are not supported for imported data", %{ + conn: conn, + site: site + } do + site_import = insert(:site_import, site: site) + + insert(:goal, event_name: "Signup", site: site) + + populate_stats(site, site_import.id, [ + build(:event, + name: "Signup", + "meta.key": ["package"], + "meta.value": ["large"] + ), + build(:imported_visitors, visitors: 9) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:package"], + "filters" => [ + ["is", "event:goal", ["Signup"]] + ], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["large"], "metrics" => [1]} + ] + + assert json_response(conn, 200)["meta"]["warning"] =~ + "Imported stats are not included in the results because query parameters are not supported." + end + + test "does not add a warning when there are no site imports", %{conn: conn, site: site} do + insert(:goal, event_name: "Signup", site: site) + + populate_stats(site, [ + build(:event, + name: "Signup", + "meta.key": ["package"], + "meta.value": ["large"] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:package"], + "filters" => [ + ["is", "event:goal", ["Signup"]] + ], + "include" => %{"imports" => true} + }) + + refute json_response(conn, 200)["meta"]["warning"] + end + + test "does not add a warning when import is out of queried date range", %{ + conn: conn, + site: site + } do + site_import = insert(:site_import, site: site, end_date: Date.add(Date.utc_today(), -3)) + + insert(:goal, event_name: "Signup", site: site) + + populate_stats(site, site_import.id, [ + build(:event, + name: "Signup", + "meta.key": ["package"], + "meta.value": ["large"] + ), + build(:imported_visitors, visitors: 9) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "day", + "dimensions" => ["event:props:package"], + "filters" => [ + ["is", "event:goal", ["Signup"]] + ], + "include" => %{"imports" => true} + }) + + refute json_response(conn, 200)["meta"]["warning"] + end + + test "applies multiple filters if the properties belong to the same table", %{ + conn: conn, + site: site + } do + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:imported_sources, source: "Google", utm_medium: "organic", utm_term: "one"), + build(:imported_sources, source: "Twitter", utm_medium: "organic", utm_term: "two"), + build(:imported_sources, + source: "Facebook", + utm_medium: "something_else", + utm_term: "one" + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "day", + "dimensions" => ["visit:source"], + "filters" => [ + ["is", "visit:utm_medium", ["organic"]], + ["is", "visit:utm_term", ["one"]] + ], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => ["Google"], "metrics" => [1]} + ] + end + + test "ignores imported data if filtered property belongs to a different table than the breakdown property", + %{ + conn: conn, + site: site + } do + site_import = insert(:site_import, site: site) + + populate_stats(site, site_import.id, [ + build(:imported_sources, source: "Google"), + build(:imported_devices, device: "Desktop") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "day", + "dimensions" => ["visit:source"], + "filters" => [ + ["is", "visit:device", ["Desktop"]] + ], + "include" => %{"imports" => true} + }) + + assert %{ + "results" => [], + "meta" => meta + } = json_response(conn, 200) + + assert meta["warning"] =~ "Imported stats are not included in the results" + end + end +end