plausible/analytics
1
1
Fork 0
mirror of https://github.com/plausible/analytics.git synced 2024-09-11 18:07:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add event:goal property to Stats API (#3718)

Browse Source 
* add tests for filtering by goal in timeseries and aggregate

* refactor filter parsing

* stop returning custom props in event:goal breakdown

* test breaking down wildcard pageview goals

* extract filter utils

* parse more goal filter options

* add passing tests for new filter types

* do not allow querying session metrics with a goal filter

* remove unused page_match property

* test that non-configured goals are not returned in breakdown

* enforce filtered goals configured

* update changelog

* Allow simple filtering by revenue goals

This does not mean that revenue metrics are supported. If a revenue goal
is filtered by, we treat it like a simple custom event goal in the API.

* use List.wrap
This commit is contained in:
RobertJoonas 2024-01-29 10:16:47 +00:00 committed by GitHub
parent 6d229b3f70
commit 04b9067591
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
21 changed files with 1173 additions and 623 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -2,6 +2,7 @@
All notable changes to this project will be documented in this file.
### Added
- Wildcard and member filtering on the Stats API `event:goal` property
- Allow filtering with `contains`/`matches` operator for custom properties
- Add `referrers.csv` to CSV export
- Add a new Properties section in the dashboard to break down by custom properties
@ -21,6 +22,7 @@ All notable changes to this project will be documented in this file.
### Removed
- Removed the nested custom event property breakdown UI when filtering by a goal in Goal Conversions
- Removed the `prop_names` returned in the Stats API `event:goal` breakdown response
- Removed the `prop-breakdown.csv` file from CSV export
- Deprecated `CLICKHOUSE_MAX_BUFFER_SIZE`

18
lib/plausible/stats/breakdown.ex
View File

@ -436,24 +436,6 @@ defmodule Plausible.Stats.Breakdown do
)
end
defp do_group_by(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,
"event:page_match"
) do
case Map.get(q, :__private_match_sources__) do
match_exprs when is_list(match_exprs) ->
from(
e in q,
group_by: fragment("index"),
select_merge: %{
index: fragment("arrayJoin(indices) as index"),
page_match: fragment("?[index]", ^match_exprs)
},
order_by: {:asc, fragment("index")}
)
end
end
defp do_group_by(q, "visit:source") do
from(
s in q,

15
lib/plausible/stats/custom_props.ex
View File

@ -7,21 +7,6 @@ defmodule Plausible.Stats.CustomProps do
use Plausible.ClickhouseRepo
import Plausible.Stats.Base
@doc """
Returns a breakdown of event names with all existing custom
properties for each event name.
"""
def props_for_all_event_names(site, query) do
from(e in base_event_query(site, query),
array_join: meta in fragment("meta"),
group_by: e.name,
select: {e.name, fragment("groupArray(?)", meta.key)},
distinct: true
)
|> ClickhouseRepo.all()
|> Enum.into(%{})
end
def fetch_prop_names(site, query) do
case Query.get_filter_by_prefix(query, "event:props:") do
{"event:props:" <> key, _} ->

102
lib/plausible/stats/filter_parser.ex
View File

@ -1,102 +0,0 @@
defmodule Plausible.Stats.FilterParser do
@moduledoc """
A module for parsing filters used in stat queries.
"""
@doc """
Parses different filter formats.
Depending on the format and type of the `filters` argument, returns:
* a decoded map, when `filters` is encoded JSON
* a parsed filter map, when `filters` is a filter expression string
* the same map, when `filters` is a map
Returns an empty map when argument type is unexpected (e.g. `nil`).
### Examples:
iex> FilterParser.parse_filters("{\\"page\\":\\"/blog/**\\"}")
%{"page" => "/blog/**"}
iex> FilterParser.parse_filters("visit:browser!=Chrome")
%{"visit:browser" => {:is_not, "Chrome"}}
iex> FilterParser.parse_filters(nil)
%{}
"""
def parse_filters(filters) when is_binary(filters) do
case Jason.decode(filters) do
{:ok, parsed} when is_map(parsed) -> parsed
{:ok, _} -> %{}
{:error, err} -> parse_filter_expression(err.data)
end
end
def parse_filters(filters) when is_map(filters), do: filters
def parse_filters(_), do: %{}
defp parse_filter_expression(str) do
filters = String.split(str, ";")
Enum.map(filters, &parse_single_filter/1)
|> Enum.reject(fn parsed -> parsed == :error end)
|> Enum.into(%{})
end
@non_escaped_pipe_regex ~r/(?<!\\)\|/
defp parse_single_filter(str) do
case to_kv(str) do
[key, raw_value] ->
is_negated = String.contains?(str, "!=")
is_list = Regex.match?(@non_escaped_pipe_regex, raw_value)
is_wildcard = String.contains?(raw_value, "*")
final_value = remove_escape_chars(raw_value)
cond do
key == "event:goal" -> {key, parse_goal_filter(final_value)}
is_wildcard && is_negated -> {key, {:does_not_match, raw_value}}
is_wildcard -> {key, {:matches, raw_value}}
is_list -> {key, {:member, parse_member_list(raw_value)}}
is_negated -> {key, {:is_not, final_value}}
true -> {key, {:is, final_value}}
end
|> reject_invalid_country_codes()
_ ->
:error
end
end
defp reject_invalid_country_codes({"visit:country", {_, code_or_codes}} = filter) do
code_or_codes
|> List.wrap()
|> Enum.reduce_while(filter, fn
value, _ when byte_size(value) == 2 -> {:cont, filter}
_, _ -> {:halt, :error}
end)
end
defp reject_invalid_country_codes(filter), do: filter
defp to_kv(str) do
str
|> String.trim()
|> String.split(["==", "!="], trim: true)
|> Enum.map(&String.trim/1)
end
defp parse_goal_filter("Visit " <> page), do: {:is, {:page, page}}
defp parse_goal_filter(event), do: {:is, {:event, event}}
defp remove_escape_chars(value) do
String.replace(value, "\\|", "|")
end
defp parse_member_list(raw_value) do
raw_value
|> String.split(@non_escaped_pipe_regex)
|> Enum.map(&remove_escape_chars/1)
end
end

101
lib/plausible/stats/filters.ex
View File

@ -1,101 +0,0 @@
defmodule Plausible.Stats.Filters do
@visit_props [
"source",
"referrer",
"utm_medium",
"utm_source",
"utm_campaign",
"utm_content",
"utm_term",
"screen",
"device",
"browser",
"browser_version",
"os",
"os_version",
"country",
"region",
"city",
"entry_page",
"exit_page"
]
@event_props [
"name",
"page",
"goal"
]
def visit_props() do
@visit_props
end
def add_prefix(query) do
new_filters =
Enum.reduce(query.filters, %{}, fn {name, val}, new_filters ->
cond do
name in @visit_props ->
Map.put(new_filters, "visit:" <> name, filter_value(name, val))
name in @event_props ->
Map.put(new_filters, "event:" <> name, filter_value(name, val))
name == "props" ->
Enum.reduce(val, new_filters, fn {prop_key, prop_val}, new_filters ->
Map.put(new_filters, "event:props:" <> prop_key, filter_value(name, prop_val))
end)
true ->
new_filters
end
end)
%Plausible.Stats.Query{query | filters: new_filters}
end
@non_escaped_pipe_regex ~r/(?<!\\)\|/
defp filter_value(key, val) do
{is_negated, val} = parse_negated_prefix(val)
{is_contains, val} = parse_contains_prefix(val)
is_list = Regex.match?(@non_escaped_pipe_regex, val)
is_wildcard = String.contains?(key, ["page", "goal"]) && String.match?(val, ~r/\*/)
val = if is_list, do: parse_member_list(val), else: remove_escape_chars(val)
val = if key == "goal", do: wrap_goal_value(val), else: val
cond do
is_negated && is_wildcard && is_list -> {:not_matches_member, val}
is_negated && is_contains && is_list -> {:not_matches_member, Enum.map(val, &"**#{&1}**")}
is_wildcard && is_list -> {:matches_member, val}
is_negated && is_wildcard -> {:does_not_match, val}
is_negated && is_list -> {:not_member, val}
is_negated && is_contains -> {:does_not_match, "**" <> val <> "**"}
is_contains && is_list -> {:matches_member, Enum.map(val, &"**#{&1}**")}
is_wildcard && is_list -> {:matches_member, val}
is_negated -> {:is_not, val}
is_list -> {:member, val}
is_contains -> {:matches, "**" <> val <> "**"}
is_wildcard -> {:matches, val}
true -> {:is, val}
end
end
defp parse_negated_prefix("!" <> val), do: {true, val}
defp parse_negated_prefix(val), do: {false, val}
defp parse_contains_prefix("~" <> val), do: {true, val}
defp parse_contains_prefix(val), do: {false, val}
defp parse_member_list(raw_value) do
raw_value
|> String.split(@non_escaped_pipe_regex)
|> Enum.map(&remove_escape_chars/1)
end
defp remove_escape_chars(value) do
String.replace(value, "\\|", "|")
end
defp wrap_goal_value(goals) when is_list(goals), do: Enum.map(goals, &wrap_goal_value/1)
defp wrap_goal_value("Visit " <> page), do: {:page, page}
defp wrap_goal_value(event), do: {:event, event}
end

65
lib/plausible/stats/filters/dashboard_filter_parser.ex Normal file
View File

@ -0,0 +1,65 @@
defmodule Plausible.Stats.Filters.DashboardFilterParser do
@moduledoc false
import Plausible.Stats.Filters.Utils
alias Plausible.Stats.Filters
@doc """
This function parses and prefixes the map filter format used by
the internal React dashboard API
"""
def parse_and_prefix(filters_map) do
Enum.reduce(filters_map, %{}, fn {name, val}, new_filters ->
cond do
name in Filters.visit_props() ->
Map.put(new_filters, "visit:" <> name, filter_value(name, val))
name in Filters.event_props() ->
Map.put(new_filters, "event:" <> name, filter_value(name, val))
name == "props" ->
put_parsed_props(new_filters, name, val)
true ->
new_filters
end
end)
end
defp put_parsed_props(new_filters, name, val) do
Enum.reduce(val, new_filters, fn {prop_key, prop_val}, new_filters ->
Map.put(new_filters, "event:props:" <> prop_key, filter_value(name, prop_val))
end)
end
defp filter_value(key, val) do
{is_negated, val} = parse_negated_prefix(val)
{is_contains, val} = parse_contains_prefix(val)
is_list = list_expression?(val)
is_wildcard = String.contains?(key, ["page", "goal"]) && wildcard_expression?(val)
val = if is_list, do: parse_member_list(val), else: remove_escape_chars(val)
val = if key == "goal", do: wrap_goal_value(val), else: val
cond do
is_negated && is_wildcard && is_list -> {:not_matches_member, val}
is_negated && is_contains && is_list -> {:not_matches_member, Enum.map(val, &"**#{&1}**")}
is_wildcard && is_list -> {:matches_member, val}
is_negated && is_wildcard -> {:does_not_match, val}
is_negated && is_list -> {:not_member, val}
is_negated && is_contains -> {:does_not_match, "**" <> val <> "**"}
is_contains && is_list -> {:matches_member, Enum.map(val, &"**#{&1}**")}
is_wildcard && is_list -> {:matches_member, val}
is_negated -> {:is_not, val}
is_list -> {:member, val}
is_contains -> {:matches, "**" <> val <> "**"}
is_wildcard -> {:matches, val}
true -> {:is, val}
end
end
defp parse_negated_prefix("!" <> val), do: {true, val}
defp parse_negated_prefix(val), do: {false, val}
defp parse_contains_prefix("~" <> val), do: {true, val}
defp parse_contains_prefix(val), do: {false, val}
end

70
lib/plausible/stats/filters/filters.ex Normal file
View File

@ -0,0 +1,70 @@
defmodule Plausible.Stats.Filters do
@moduledoc """
A module for parsing filters used in stat queries.
"""
alias Plausible.Stats.Filters.{DashboardFilterParser, StatsAPIFilterParser}
@visit_props [
"source",
"referrer",
"utm_medium",
"utm_source",
"utm_campaign",
"utm_content",
"utm_term",
"screen",
"device",
"browser",
"browser_version",
"os",
"os_version",
"country",
"region",
"city",
"entry_page",
"exit_page"
]
def visit_props(), do: @visit_props
@event_props [
"name",
"page",
"goal"
]
def event_props(), do: @event_props
@doc """
Parses different filter formats.
Depending on the format and type of the `filters` argument, returns:
* a decoded map, when `filters` is encoded JSON
* a parsed filter map, when `filters` is a filter expression string
* the same map, when `filters` is a map
Returns an empty map when argument type is unexpected (e.g. `nil`).
### Examples:
iex> Filters.parse("{\\"page\\":\\"/blog/**\\"}")
%{"event:page" => {:matches, "/blog/**"}}
iex> Filters.parse("visit:browser!=Chrome")
%{"visit:browser" => {:is_not, "Chrome"}}
iex> Filters.parse(nil)
%{}
"""
def parse(filters) when is_binary(filters) do
case Jason.decode(filters) do
{:ok, filters} when is_map(filters) -> DashboardFilterParser.parse_and_prefix(filters)
{:ok, _} -> %{}
{:error, err} -> StatsAPIFilterParser.parse_filter_expression(err.data)
end
end
def parse(filters) when is_map(filters), do: DashboardFilterParser.parse_and_prefix(filters)
def parse(_), do: %{}
end

81
lib/plausible/stats/filters/stats_api_filter_parser.ex Normal file
View File

@ -0,0 +1,81 @@
defmodule Plausible.Stats.Filters.StatsAPIFilterParser do
@moduledoc false
import Plausible.Stats.Filters.Utils
@doc """
This function parses the filter expression given as a string.
This filtering format is used by the public Stats API.
"""
def parse_filter_expression(str) do
filters = String.split(str, ";")
Enum.map(filters, &parse_single_filter/1)
|> Enum.reject(fn parsed -> parsed == :error end)
|> Enum.into(%{})
end
defp parse_single_filter(str) do
case to_kv(str) do
["event:goal" = key, raw_value] ->
{key, parse_goal_filter(raw_value)}
[key, raw_value] ->
is_negated? = String.contains?(str, "!=")
is_list? = list_expression?(raw_value)
is_wildcard? = wildcard_expression?(raw_value)
final_value = remove_escape_chars(raw_value)
cond do
is_wildcard? && is_negated? -> {key, {:does_not_match, raw_value}}
is_wildcard? -> {key, {:matches, raw_value}}
is_list? -> {key, {:member, parse_member_list(raw_value)}}
is_negated? -> {key, {:is_not, final_value}}
true -> {key, {:is, final_value}}
end
|> reject_invalid_country_codes()
_ ->
:error
end
end
defp reject_invalid_country_codes({"visit:country", {_, code_or_codes}} = filter) do
code_or_codes
|> List.wrap()
|> Enum.reduce_while(filter, fn
value, _ when byte_size(value) == 2 -> {:cont, filter}
_, _ -> {:halt, :error}
end)
end
defp reject_invalid_country_codes(filter), do: filter
defp to_kv(str) do
str
|> String.trim()
|> String.split(["==", "!="], trim: true)
|> Enum.map(&String.trim/1)
end
defp parse_goal_filter(value) do
is_list? = list_expression?(value)
is_wildcard? = wildcard_expression?(value)
value =
if is_list? do
parse_member_list(value)
else
remove_escape_chars(value)
end
|> wrap_goal_value()
cond do
is_list? && is_wildcard? -> {:matches_member, value}
is_list? -> {:member, value}
is_wildcard? -> {:matches, value}
true -> {:is, value}
end
end
end

69
lib/plausible/stats/filters/utils.ex Normal file
View File

@ -0,0 +1,69 @@
defmodule Plausible.Stats.Filters.Utils do
@moduledoc """
Contains utility functions shared between `DashboardFilterParser`
and `StatsAPIFilterParser`.
"""
@non_escaped_pipe_regex ~r/(?<!\\)\|/
def list_expression?(expression) do
Regex.match?(@non_escaped_pipe_regex, expression)
end
def wildcard_expression?(expression) do
String.contains?(expression, "*")
end
def parse_member_list(raw_value) do
raw_value
|> String.split(@non_escaped_pipe_regex)
|> Enum.map(&remove_escape_chars/1)
end
def remove_escape_chars(value) do
String.replace(value, "\\|", "|")
end
@doc """
Wraps the given goal filter into a tuple where the first element
represents the goal type (i.e. `page` or `event`), and the second
element is either the page path or the event name.
Given a list of goal filter values, wraps all values based on the
same logic.
## Examples
iex> Plausible.Stats.Filters.Utils.wrap_goal_value("Visit /register")
{:page, "/register"}
iex> Plausible.Stats.Filters.Utils.wrap_goal_value("Signup")
{:event, "Signup"}
iex> Plausible.Stats.Filters.Utils.wrap_goal_value(["Signup", "Visit /register"])
[{:event, "Signup"}, {:page, "/register"}]
"""
def wrap_goal_value(goals) when is_list(goals), do: Enum.map(goals, &wrap_goal_value/1)
def wrap_goal_value("Visit " <> page), do: {:page, page}
def wrap_goal_value(event), do: {:event, event}
@doc """
Does the opposite to `wrap_goal_value`, turning the `{:page, path}`
and `{:event, name}` tuples into strings. Similarly, when given a
list, maps all the list elements into strings with the same logic.
## Examples
iex> Plausible.Stats.Filters.Utils.unwrap_goal_value({:page, "/register"})
"Visit /register"
iex> Plausible.Stats.Filters.Utils.unwrap_goal_value({:event, "Signup"})
"Signup"
iex> Plausible.Stats.Filters.Utils.unwrap_goal_value([{:event, "Signup"}, {:page, "/register"}])
["Signup", "Visit /register"]
"""
def unwrap_goal_value(goals) when is_list(goals), do: Enum.map(goals, &unwrap_goal_value/1)
def unwrap_goal_value({:page, page}), do: "Visit " <> page
def unwrap_goal_value({:event, event}), do: event
end

2
lib/plausible/stats/props.ex
View File

@ -1,5 +1,5 @@
defmodule Plausible.Stats.Props do
@event_props ["event:page", "event:page_match", "event:name", "event:goal"]
@event_props ["event:page", "event:name", "event:goal"]
@session_props [
"visit:source",
"visit:country",

4
lib/plausible/stats/query.ex
View File

@ -11,7 +11,7 @@ defmodule Plausible.Stats.Query do
now: nil
require OpenTelemetry.Tracer, as: Tracer
alias Plausible.Stats.{FilterParser, Interval}
alias Plausible.Stats.{Filters, Interval}
@type t :: %__MODULE__{}
@ -164,7 +164,7 @@ defmodule Plausible.Stats.Query do
end
defp put_parsed_filters(query, params) do
struct!(query, filters: FilterParser.parse_filters(params["filters"]))
struct!(query, filters: Filters.parse(params["filters"]))
end
def put_filter(query, key, val) do

50
lib/plausible_web/controllers/api/external_stats_controller.ex
View File

@ -2,7 +2,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
use PlausibleWeb, :controller
use Plausible.Repo
use PlausibleWeb.Plugs.ErrorHandler
alias Plausible.Stats.{Query, CustomProps}
alias Plausible.Stats.Query
def realtime_visitors(conn, _params) do
site = conn.assigns.site
@ -16,6 +16,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
with :ok <- validate_period(params),
:ok <- validate_date(params),
query <- Query.from(site, params),
:ok <- validate_goal_filter(site, query.filters),
{:ok, metrics} <- parse_and_validate_metrics(params, nil, query),
:ok <- ensure_custom_props_access(site, query) do
results =
@ -55,23 +56,13 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
:ok <- validate_date(params),
{:ok, property} <- validate_property(params),
query <- Query.from(site, params),
:ok <- validate_goal_filter(site, query.filters),
{:ok, metrics} <- parse_and_validate_metrics(params, property, query),
{:ok, limit} <- validate_or_default_limit(params),
:ok <- ensure_custom_props_access(site, query, property) do
page = String.to_integer(Map.get(params, "page", "1"))
results = Plausible.Stats.breakdown(site, query, property, metrics, {limit, page})
results =
if property == "event:goal" do
prop_names = CustomProps.props_for_all_event_names(site, query)
Enum.map(results, fn row ->
Map.put(row, "props", prop_names[row[:goal]] || [])
end)
else
results
end
json(conn, %{results: results})
else
err_tuple -> send_json_error_response(conn, err_tuple)
@ -106,6 +97,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
defp validate_or_default_limit(_), do: {:ok, @default_breakdown_limit}
defp event_only_property?("event:name"), do: true
defp event_only_property?("event:goal"), do: true
defp event_only_property?("event:props:" <> _), do: true
defp event_only_property?(_), do: false
@ -203,6 +195,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
:ok <- validate_date(params),
:ok <- validate_interval(params),
query <- Query.from(site, params),
:ok <- validate_goal_filter(site, query.filters),
{:ok, metrics} <- parse_and_validate_metrics(params, nil, query),
:ok <- ensure_custom_props_access(site, query) do
graph = Plausible.Stats.timeseries(site, query, metrics)
@ -281,6 +274,39 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
defp validate_interval(_), do: :ok
defp validate_goal_filter(site, %{"event:goal" => {_type, goal_filter}}) do
configured_goals =
Plausible.Goals.for_site(site)
|> Enum.map(fn
%{page_path: path} when is_binary(path) -> "Visit " <> path
%{event_name: event_name} -> event_name
end)
goals_in_filter =
List.wrap(goal_filter)
|> Plausible.Stats.Filters.Utils.unwrap_goal_value()
if found = Enum.find(goals_in_filter, &(&1 not in configured_goals)) do
msg =
goal_not_configured_message(found) <>
"Find out how to configure goals here: https://plausible.io/docs/stats-api#filtering-by-goals"
{:error, msg}
else
:ok
end
end
defp validate_goal_filter(_site, _filters), do: :ok
defp goal_not_configured_message("Visit " <> page_path) do
"The pageview goal for the pathname `#{page_path}` is not configured for this site. "
end
defp goal_not_configured_message(goal) do
"The goal `#{goal}` is not configured for this site. "
end
defp send_json_error_response(conn, {:error, {status, msg}}) do
conn
|> put_status(status)

73
lib/plausible_web/controllers/api/stats_controller.ex
View File

@ -4,7 +4,7 @@ defmodule PlausibleWeb.Api.StatsController do
use Plausible.Repo
use PlausibleWeb.Plugs.ErrorHandler
alias Plausible.Stats
alias Plausible.Stats.{Query, Filters, Comparisons}
alias Plausible.Stats.{Query, Comparisons}
alias PlausibleWeb.Api.Helpers, as: H
require Logger
@ -99,7 +99,7 @@ defmodule PlausibleWeb.Api.StatsController do
site = conn.assigns[:site]
with :ok <- validate_params(site, params) do
query = Query.from(site, params) |> Filters.add_prefix()
query = Query.from(site, params)
selected_metric =
if !params["metric"] || params["metric"] == "conversions" do
@ -204,7 +204,7 @@ defmodule PlausibleWeb.Api.StatsController do
site = conn.assigns[:site]
with :ok <- validate_params(site, params) do
query = Query.from(site, params) |> Filters.add_prefix()
query = Query.from(site, params)
comparison_opts = parse_comparison_opts(params)
@ -464,9 +464,7 @@ defmodule PlausibleWeb.Api.StatsController do
def sources(conn, params) do
site = conn.assigns[:site]
query =
Query.from(site, params)
|> Filters.add_prefix()
query = Query.from(site, params)
pagination = parse_pagination(params)
@ -497,7 +495,7 @@ defmodule PlausibleWeb.Api.StatsController do
with :ok <- Plausible.Billing.Feature.Funnels.check_availability(site.owner),
:ok <- validate_params(site, params),
query <- Query.from(site, params) |> Filters.add_prefix(),
query <- Query.from(site, params),
:ok <- validate_funnel_query(query),
{funnel_id, ""} <- Integer.parse(funnel_id),
{:ok, funnel} <- Stats.funnel(site, query, funnel_id) do
@ -549,9 +547,7 @@ defmodule PlausibleWeb.Api.StatsController do
def utm_mediums(conn, params) do
site = conn.assigns[:site]
query =
Query.from(site, params)
|> Filters.add_prefix()
query = Query.from(site, params)
pagination = parse_pagination(params)
@ -578,9 +574,7 @@ defmodule PlausibleWeb.Api.StatsController do
def utm_campaigns(conn, params) do
site = conn.assigns[:site]
query =
Query.from(site, params)
|> Filters.add_prefix()
query = Query.from(site, params)
pagination = parse_pagination(params)
@ -607,9 +601,7 @@ defmodule PlausibleWeb.Api.StatsController do
def utm_contents(conn, params) do
site = conn.assigns[:site]
query =
Query.from(site, params)
|> Filters.add_prefix()
query = Query.from(site, params)
pagination = parse_pagination(params)
metrics = [:visitors, :bounce_rate, :visit_duration]
@ -635,9 +627,7 @@ defmodule PlausibleWeb.Api.StatsController do
def utm_terms(conn, params) do
site = conn.assigns[:site]
query =
Query.from(site, params)
|> Filters.add_prefix()
query = Query.from(site, params)
pagination = parse_pagination(params)
metrics = [:visitors, :bounce_rate, :visit_duration]
@ -663,9 +653,7 @@ defmodule PlausibleWeb.Api.StatsController do
def utm_sources(conn, params) do
site = conn.assigns[:site]
query =
Query.from(site, params)
|> Filters.add_prefix()
query = Query.from(site, params)
pagination = parse_pagination(params)
@ -692,9 +680,7 @@ defmodule PlausibleWeb.Api.StatsController do
def referrers(conn, params) do
site = conn.assigns[:site]
query =
Query.from(site, params)
|> Filters.add_prefix()
query = Query.from(site, params)
pagination = parse_pagination(params)
@ -723,8 +709,7 @@ defmodule PlausibleWeb.Api.StatsController do
query =
Query.from(site, params)
|> Query.put_filter("source", "Google")
|> Filters.add_prefix()
|> Query.put_filter("visit:source", {:is, "Google"})
search_terms =
if site.google_auth && site.google_auth.property && !query.filters["goal"] do
@ -759,8 +744,7 @@ defmodule PlausibleWeb.Api.StatsController do
query =
Query.from(site, params)
|> Query.put_filter("source", referrer)
|> Filters.add_prefix()
|> Query.put_filter("visit:source", {:is, referrer})
pagination = parse_pagination(params)
@ -777,7 +761,7 @@ defmodule PlausibleWeb.Api.StatsController do
def pages(conn, params) do
site = conn.assigns[:site]
query = Query.from(site, params) |> Filters.add_prefix()
query = Query.from(site, params)
metrics =
if params["detailed"],
@ -806,7 +790,7 @@ defmodule PlausibleWeb.Api.StatsController do
def entry_pages(conn, params) do
site = conn.assigns[:site]
query = Query.from(site, params) |> Filters.add_prefix()
query = Query.from(site, params)
pagination = parse_pagination(params)
metrics = [:visitors, :visits, :visit_duration]
@ -837,7 +821,7 @@ defmodule PlausibleWeb.Api.StatsController do
def exit_pages(conn, params) do
site = conn.assigns[:site]
query = Query.from(site, params) |> Filters.add_prefix()
query = Query.from(site, params)
{limit, page} = parse_pagination(params)
metrics = [:visitors, :visits]
@ -897,7 +881,7 @@ defmodule PlausibleWeb.Api.StatsController do
def countries(conn, params) do
site = conn.assigns[:site]
query = site |> Query.from(params) |> Filters.add_prefix()
query = site |> Query.from(params)
pagination = parse_pagination(params)
countries =
@ -949,7 +933,7 @@ defmodule PlausibleWeb.Api.StatsController do
def regions(conn, params) do
site = conn.assigns[:site]
query = site |> Query.from(params) |> Filters.add_prefix()
query = site |> Query.from(params)
pagination = parse_pagination(params)
regions =
@ -982,7 +966,7 @@ defmodule PlausibleWeb.Api.StatsController do
def cities(conn, params) do
site = conn.assigns[:site]
query = site |> Query.from(params) |> Filters.add_prefix()
query = site |> Query.from(params)
pagination = parse_pagination(params)
cities =
@ -1020,7 +1004,7 @@ defmodule PlausibleWeb.Api.StatsController do
def browsers(conn, params) do
site = conn.assigns[:site]
query = Query.from(site, params) |> Filters.add_prefix()
query = Query.from(site, params)
pagination = parse_pagination(params)
browsers =
@ -1044,7 +1028,7 @@ defmodule PlausibleWeb.Api.StatsController do
def browser_versions(conn, params) do
site = conn.assigns[:site]
query = Query.from(site, params) |> Filters.add_prefix()
query = Query.from(site, params)
pagination = parse_pagination(params)
versions =
@ -1074,7 +1058,7 @@ defmodule PlausibleWeb.Api.StatsController do
def operating_systems(conn, params) do
site = conn.assigns[:site]
query = Query.from(site, params) |> Filters.add_prefix()
query = Query.from(site, params)
pagination = parse_pagination(params)
systems =
@ -1098,7 +1082,7 @@ defmodule PlausibleWeb.Api.StatsController do
def operating_system_versions(conn, params) do
site = conn.assigns[:site]
query = Query.from(site, params) |> Filters.add_prefix()
query = Query.from(site, params)
pagination = parse_pagination(params)
versions =
@ -1112,7 +1096,7 @@ defmodule PlausibleWeb.Api.StatsController do
def screen_sizes(conn, params) do
site = conn.assigns[:site]
query = Query.from(site, params) |> Filters.add_prefix()
query = Query.from(site, params)
pagination = parse_pagination(params)
sizes =
@ -1145,7 +1129,7 @@ defmodule PlausibleWeb.Api.StatsController do
def conversions(conn, params) do
pagination = parse_pagination(params)
site = Plausible.Repo.preload(conn.assigns.site, :goals)
query = Query.from(site, params) |> Filters.add_prefix()
query = Query.from(site, params)
query =
if query.period == "realtime" do
@ -1210,7 +1194,7 @@ defmodule PlausibleWeb.Api.StatsController do
def all_custom_prop_values(conn, params) do
site = conn.assigns.site
query = Query.from(site, params) |> Filters.add_prefix()
query = Query.from(site, params)
prop_names = Plausible.Stats.CustomProps.fetch_prop_names(site, query)
@ -1246,7 +1230,6 @@ defmodule PlausibleWeb.Api.StatsController do
query =
Query.from(site, params)
|> Filters.add_prefix()
|> Map.put(:include_imported, false)
metrics =
@ -1288,9 +1271,7 @@ defmodule PlausibleWeb.Api.StatsController do
def filter_suggestions(conn, params) do
site = conn.assigns[:site]
query =
Query.from(site, params)
|> Filters.add_prefix()
query = Query.from(site, params)
json(
conn,

4
lib/plausible_web/controllers/stats_controller.ex
View File

@ -45,7 +45,7 @@ defmodule PlausibleWeb.StatsController do
use Plausible.Repo
alias Plausible.Sites
alias Plausible.Stats.{Query, Filters}
alias Plausible.Stats.Query
alias PlausibleWeb.Api
plug(PlausibleWeb.AuthorizeSiteAccess when action in [:stats, :csv_export])
@ -110,7 +110,7 @@ defmodule PlausibleWeb.StatsController do
def csv_export(conn, params) do
if is_nil(params["interval"]) or Plausible.Stats.Interval.valid?(params["interval"]) do
site = Plausible.Repo.preload(conn.assigns.site, :owner)
query = Query.from(site, params) |> Filters.add_prefix()
query = Query.from(site, params)
metrics =
if query.filters["event:goal"] do

204
test/plausible/stats/dashboard_filter_parser_test.exs Normal file
View File

@ -0,0 +1,204 @@
defmodule Plausible.Stats.DashboardFilterParserTest do
use ExUnit.Case, async: true
alias Plausible.Stats.Filters.DashboardFilterParser
def assert_parsed(filters, expected_output) do
assert DashboardFilterParser.parse_and_prefix(filters) == expected_output
end
describe "adding prefix" do
test "adds appropriate prefix to filter" do
%{"page" => "/"}
|> assert_parsed(%{"event:page" => {:is, "/"}})
%{"goal" => "Signup"}
|> assert_parsed(%{"event:goal" => {:is, {:event, "Signup"}}})
%{"goal" => "Visit /blog"}
|> assert_parsed(%{"event:goal" => {:is, {:page, "/blog"}}})
%{"source" => "Google"}
|> assert_parsed(%{"visit:source" => {:is, "Google"}})
%{"referrer" => "cnn.com"}
|> assert_parsed(%{"visit:referrer" => {:is, "cnn.com"}})
%{"utm_medium" => "search"}
|> assert_parsed(%{"visit:utm_medium" => {:is, "search"}})
%{"utm_source" => "bing"}
|> assert_parsed(%{"visit:utm_source" => {:is, "bing"}})
%{"utm_content" => "content"}
|> assert_parsed(%{"visit:utm_content" => {:is, "content"}})
%{"utm_term" => "term"}
|> assert_parsed(%{"visit:utm_term" => {:is, "term"}})
%{"screen" => "Desktop"}
|> assert_parsed(%{"visit:screen" => {:is, "Desktop"}})
%{"browser" => "Opera"}
|> assert_parsed(%{"visit:browser" => {:is, "Opera"}})
%{"browser_version" => "10.1"}
|> assert_parsed(%{"visit:browser_version" => {:is, "10.1"}})
%{"os" => "Linux"}
|> assert_parsed(%{"visit:os" => {:is, "Linux"}})
%{"os_version" => "13.0"}
|> assert_parsed(%{"visit:os_version" => {:is, "13.0"}})
%{"country" => "EE"}
|> assert_parsed(%{"visit:country" => {:is, "EE"}})
%{"region" => "EE-12"}
|> assert_parsed(%{"visit:region" => {:is, "EE-12"}})
%{"city" => "123"}
|> assert_parsed(%{"visit:city" => {:is, "123"}})
%{"entry_page" => "/blog"}
|> assert_parsed(%{"visit:entry_page" => {:is, "/blog"}})
%{"exit_page" => "/blog"}
|> assert_parsed(%{"visit:exit_page" => {:is, "/blog"}})
%{"props" => %{"cta" => "Top"}}
|> assert_parsed(%{"event:props:cta" => {:is, "Top"}})
end
end
describe "escaping pipe character" do
test "in simple is filter" do
%{"goal" => ~S(Foo \| Bar)}
|> assert_parsed(%{"event:goal" => {:is, {:event, "Foo | Bar"}}})
end
test "in member filter" do
%{"page" => ~S(/|\|)}
|> assert_parsed(%{"event:page" => {:member, ["/", "|"]}})
end
end
describe "is not filter type" do
test "simple is not filter" do
%{"page" => "!/"}
|> assert_parsed(%{"event:page" => {:is_not, "/"}})
%{"props" => %{"cta" => "!Top"}}
|> assert_parsed(%{"event:props:cta" => {:is_not, "Top"}})
end
end
describe "member filter type" do
test "simple member filter" do
%{"page" => "/|/blog"}
|> assert_parsed(%{"event:page" => {:member, ["/", "/blog"]}})
end
test "escaping pipe character" do
%{"page" => "/|\\|"}
|> assert_parsed(%{"event:page" => {:member, ["/", "|"]}})
end
test "mixed goals" do
%{"goal" => "Signup|Visit /thank-you"}
|> assert_parsed(%{"event:goal" => {:member, [{:event, "Signup"}, {:page, "/thank-you"}]}})
%{"goal" => "Visit /thank-you|Signup"}
|> assert_parsed(%{"event:goal" => {:member, [{:page, "/thank-you"}, {:event, "Signup"}]}})
end
end
describe "matches_member filter type" do
test "parses matches_member filter type" do
%{"page" => "/|/blog**"}
|> assert_parsed(%{"event:page" => {:matches_member, ["/", "/blog**"]}})
end
test "parses not_matches_member filter type" do
%{"page" => "!/|/blog**"}
|> assert_parsed(%{"event:page" => {:not_matches_member, ["/", "/blog**"]}})
end
end
describe "contains filter type" do
test "single contains" do
%{"page" => "~blog"}
|> assert_parsed(%{"event:page" => {:matches, "**blog**"}})
end
test "negated contains" do
%{"page" => "!~articles"}
|> assert_parsed(%{"event:page" => {:does_not_match, "**articles**"}})
end
test "contains member" do
%{"page" => "~articles|blog"}
|> assert_parsed(%{"event:page" => {:matches_member, ["**articles**", "**blog**"]}})
end
test "not contains member" do
%{"page" => "!~articles|blog"}
|> assert_parsed(%{"event:page" => {:not_matches_member, ["**articles**", "**blog**"]}})
end
end
describe "not_member filter type" do
test "simple not_member filter" do
%{"page" => "!/|/blog"}
|> assert_parsed(%{"event:page" => {:not_member, ["/", "/blog"]}})
end
test "mixed goals" do
%{"goal" => "!Signup|Visit /thank-you"}
|> assert_parsed(%{
"event:goal" => {:not_member, [{:event, "Signup"}, {:page, "/thank-you"}]}
})
%{"goal" => "!Visit /thank-you|Signup"}
|> assert_parsed(%{
"event:goal" => {:not_member, [{:page, "/thank-you"}, {:event, "Signup"}]}
})
end
end
describe "matches filter type" do
test "can be used with `goal` or `page` filters" do
%{"page" => "/blog/post-*"}
|> assert_parsed(%{"event:page" => {:matches, "/blog/post-*"}})
%{"goal" => "Visit /blog/post-*"}
|> assert_parsed(%{"event:goal" => {:matches, {:page, "/blog/post-*"}}})
end
test "other filters default to `is` even when wildcard is present" do
%{"country" => "Germa**"}
|> assert_parsed(%{"visit:country" => {:is, "Germa**"}})
end
end
describe "does_not_match filter type" do
test "can be used with `page` filter" do
%{"page" => "!/blog/post-*"}
|> assert_parsed(%{"event:page" => {:does_not_match, "/blog/post-*"}})
end
test "other filters default to is_not even when wildcard is present" do
%{"country" => "!Germa**"}
|> assert_parsed(%{"visit:country" => {:is_not, "Germa**"}})
end
end
describe "contains prefix filter type" do
test "can be used with any filter" do
%{"page" => "~/blog/post"}
|> assert_parsed(%{"event:page" => {:matches, "**/blog/post**"}})
%{"source" => "~facebook"}
|> assert_parsed(%{"visit:source" => {:matches, "**facebook**"}})
end
end
end

100
test/plausible/stats/filter_parser_test.exs
View File

@ -1,100 +0,0 @@
defmodule Plausible.Stats.FilterParserTest do
use ExUnit.Case, async: true
alias Plausible.Stats.FilterParser
doctest Plausible.Stats.FilterParser
def assert_parsed(input, expected_output) do
assert FilterParser.parse_filters(input) == expected_output
end
describe "parses filter expression" do
test "simple positive" do
"event:name==pageview"
|> assert_parsed(%{"event:name" => {:is, "pageview"}})
end
test "simple negative" do
"event:name!=pageview"
|> assert_parsed(%{"event:name" => {:is_not, "pageview"}})
end
test "whitespace is trimmed" do
" event:name == pageview "
|> assert_parsed(%{"event:name" => {:is, "pageview"}})
end
test "wildcard" do
"event:page==/blog/post-*"
|> assert_parsed(%{"event:page" => {:matches, "/blog/post-*"}})
end
test "negative wildcard" do
"event:page!=/blog/post-*"
|> assert_parsed(%{"event:page" => {:does_not_match, "/blog/post-*"}})
end
test "custom event goal" do
"event:goal==Signup"
|> assert_parsed(%{"event:goal" => {:is, {:event, "Signup"}}})
end
test "pageview goal" do
"event:goal==Visit /blog"
|> assert_parsed(%{"event:goal" => {:is, {:page, "/blog"}}})
end
test "member" do
"visit:country==FR|GB|DE"
|> assert_parsed(%{"visit:country" => {:member, ["FR", "GB", "DE"]}})
end
test "member + wildcard" do
"event:page==/blog**|/newsletter|/*/"
|> assert_parsed(%{"event:page" => {:matches, "/blog**|/newsletter|/*/"}})
end
test "combined with \";\"" do
"event:page==/blog**|/newsletter|/*/ ; visit:country==FR|GB|DE"
|> assert_parsed(%{
"event:page" => {:matches, "/blog**|/newsletter|/*/"},
"visit:country" => {:member, ["FR", "GB", "DE"]}
})
end
test "escaping pipe character" do
"utm_campaign==campaign \\| 1"
|> assert_parsed(%{"utm_campaign" => {:is, "campaign | 1"}})
end
test "escaping pipe character in member filter" do
"utm_campaign==campaign \\| 1|campaign \\| 2"
|> assert_parsed(%{"utm_campaign" => {:member, ["campaign | 1", "campaign | 2"]}})
end
test "keeps escape characters in member + wildcard filter" do
"event:page==/**\\|page|/other/page"
|> assert_parsed(%{"event:page" => {:matches, "/**\\|page|/other/page"}})
end
test "gracefully fails to parse garbage" do
"bfg10309\uff1cs1\ufe65s2\u02bas3\u02b9hjl10309"
|> assert_parsed(%{})
end
test "gracefully fails to parse garbage with double quotes" do
"\";print(md5(31337));$a=\""
|> assert_parsed(%{})
end
test "gracefully fails to parse garbage country code" do
"visit:country==AKSJSDFKJSS"
|> assert_parsed(%{})
end
test "gracefully fails to parse garbage country code (with pipes)" do
"visit:country==ET'||DBMS_PIPE.RECEIVE_MESSAGE(CHR(98)||CHR(98)||CHR(98),15)||'"
|> assert_parsed(%{})
end
end
end

261
test/plausible/stats/filters_test.exs
View File

@ -1,208 +1,101 @@
defmodule Plausible.Stats.FiltersTest do
use ExUnit.Case, async: true
alias Plausible.Stats.{Query, Filters}
alias Plausible.Stats.Filters
def assert_parsed(filters, expected_output) do
new_query =
%Query{filters: filters}
|> Filters.add_prefix()
doctest Plausible.Stats.Filters
doctest Plausible.Stats.Filters.Utils
assert new_query.filters == expected_output
def assert_parsed(input, expected_output) do
assert Filters.parse(input) == expected_output
end
describe "adding prefix" do
test "adds appropriate prefix to filter" do
%{"page" => "/"}
|> assert_parsed(%{"event:page" => {:is, "/"}})
%{"goal" => "Signup"}
|> assert_parsed(%{"event:goal" => {:is, {:event, "Signup"}}})
%{"goal" => "Visit /blog"}
|> assert_parsed(%{"event:goal" => {:is, {:page, "/blog"}}})
%{"source" => "Google"}
|> assert_parsed(%{"visit:source" => {:is, "Google"}})
%{"referrer" => "cnn.com"}
|> assert_parsed(%{"visit:referrer" => {:is, "cnn.com"}})
%{"utm_medium" => "search"}
|> assert_parsed(%{"visit:utm_medium" => {:is, "search"}})
%{"utm_source" => "bing"}
|> assert_parsed(%{"visit:utm_source" => {:is, "bing"}})
%{"utm_content" => "content"}
|> assert_parsed(%{"visit:utm_content" => {:is, "content"}})
%{"utm_term" => "term"}
|> assert_parsed(%{"visit:utm_term" => {:is, "term"}})
%{"screen" => "Desktop"}
|> assert_parsed(%{"visit:screen" => {:is, "Desktop"}})
%{"browser" => "Opera"}
|> assert_parsed(%{"visit:browser" => {:is, "Opera"}})
%{"browser_version" => "10.1"}
|> assert_parsed(%{"visit:browser_version" => {:is, "10.1"}})
%{"os" => "Linux"}
|> assert_parsed(%{"visit:os" => {:is, "Linux"}})
%{"os_version" => "13.0"}
|> assert_parsed(%{"visit:os_version" => {:is, "13.0"}})
%{"country" => "EE"}
|> assert_parsed(%{"visit:country" => {:is, "EE"}})
%{"region" => "EE-12"}
|> assert_parsed(%{"visit:region" => {:is, "EE-12"}})
%{"city" => "123"}
|> assert_parsed(%{"visit:city" => {:is, "123"}})
%{"entry_page" => "/blog"}
|> assert_parsed(%{"visit:entry_page" => {:is, "/blog"}})
%{"exit_page" => "/blog"}
|> assert_parsed(%{"visit:exit_page" => {:is, "/blog"}})
%{"props" => %{"cta" => "Top"}}
|> assert_parsed(%{"event:props:cta" => {:is, "Top"}})
end
end
describe "escaping pipe character" do
test "in simple is filter" do
%{"goal" => ~S(Foo \| Bar)}
|> assert_parsed(%{"event:goal" => {:is, {:event, "Foo | Bar"}}})
describe "parses filter expression" do
test "simple positive" do
"event:name==pageview"
|> assert_parsed(%{"event:name" => {:is, "pageview"}})
end
test "in member filter" do
%{"page" => ~S(/|\|)}
|> assert_parsed(%{"event:page" => {:member, ["/", "|"]}})
end
end
describe "is not filter type" do
test "simple is not filter" do
%{"page" => "!/"}
|> assert_parsed(%{"event:page" => {:is_not, "/"}})
%{"props" => %{"cta" => "!Top"}}
|> assert_parsed(%{"event:props:cta" => {:is_not, "Top"}})
end
end
describe "member filter type" do
test "simple member filter" do
%{"page" => "/|/blog"}
|> assert_parsed(%{"event:page" => {:member, ["/", "/blog"]}})
test "simple negative" do
"event:name!=pageview"
|> assert_parsed(%{"event:name" => {:is_not, "pageview"}})
end
test "escaping pipe character" do
%{"page" => "/|\\|"}
|> assert_parsed(%{"event:page" => {:member, ["/", "|"]}})
test "whitespace is trimmed" do
" event:name == pageview "
|> assert_parsed(%{"event:name" => {:is, "pageview"}})
end
test "mixed goals" do
%{"goal" => "Signup|Visit /thank-you"}
|> assert_parsed(%{"event:goal" => {:member, [{:event, "Signup"}, {:page, "/thank-you"}]}})
%{"goal" => "Visit /thank-you|Signup"}
|> assert_parsed(%{"event:goal" => {:member, [{:page, "/thank-you"}, {:event, "Signup"}]}})
end
end
describe "matches_member filter type" do
test "parses matches_member filter type" do
%{"page" => "/|/blog**"}
|> assert_parsed(%{"event:page" => {:matches_member, ["/", "/blog**"]}})
end
test "parses not_matches_member filter type" do
%{"page" => "!/|/blog**"}
|> assert_parsed(%{"event:page" => {:not_matches_member, ["/", "/blog**"]}})
end
end
describe "contains filter type" do
test "single contains" do
%{"page" => "~blog"}
|> assert_parsed(%{"event:page" => {:matches, "**blog**"}})
end
test "negated contains" do
%{"page" => "!~articles"}
|> assert_parsed(%{"event:page" => {:does_not_match, "**articles**"}})
end
test "contains member" do
%{"page" => "~articles|blog"}
|> assert_parsed(%{"event:page" => {:matches_member, ["**articles**", "**blog**"]}})
end
test "not contains member" do
%{"page" => "!~articles|blog"}
|> assert_parsed(%{"event:page" => {:not_matches_member, ["**articles**", "**blog**"]}})
end
end
describe "not_member filter type" do
test "simple not_member filter" do
%{"page" => "!/|/blog"}
|> assert_parsed(%{"event:page" => {:not_member, ["/", "/blog"]}})
end
test "mixed goals" do
%{"goal" => "!Signup|Visit /thank-you"}
|> assert_parsed(%{
"event:goal" => {:not_member, [{:event, "Signup"}, {:page, "/thank-you"}]}
})
%{"goal" => "!Visit /thank-you|Signup"}
|> assert_parsed(%{
"event:goal" => {:not_member, [{:page, "/thank-you"}, {:event, "Signup"}]}
})
end
end
describe "matches filter type" do
test "can be used with `goal` or `page` filters" do
%{"page" => "/blog/post-*"}
test "wildcard" do
"event:page==/blog/post-*"
|> assert_parsed(%{"event:page" => {:matches, "/blog/post-*"}})
%{"goal" => "Visit /blog/post-*"}
|> assert_parsed(%{"event:goal" => {:matches, {:page, "/blog/post-*"}}})
end
test "other filters default to `is` even when wildcard is present" do
%{"country" => "Germa**"}
|> assert_parsed(%{"visit:country" => {:is, "Germa**"}})
end
end
describe "does_not_match filter type" do
test "can be used with `page` filter" do
%{"page" => "!/blog/post-*"}
test "negative wildcard" do
"event:page!=/blog/post-*"
|> assert_parsed(%{"event:page" => {:does_not_match, "/blog/post-*"}})
end
test "other filters default to is_not even when wildcard is present" do
%{"country" => "!Germa**"}
|> assert_parsed(%{"visit:country" => {:is_not, "Germa**"}})
test "custom event goal" do
"event:goal==Signup"
|> assert_parsed(%{"event:goal" => {:is, {:event, "Signup"}}})
end
end
describe "contains prefix filter type" do
test "can be used with any filter" do
%{"page" => "~/blog/post"}
|> assert_parsed(%{"event:page" => {:matches, "**/blog/post**"}})
test "pageview goal" do
"event:goal==Visit /blog"
|> assert_parsed(%{"event:goal" => {:is, {:page, "/blog"}}})
end
%{"source" => "~facebook"}
|> assert_parsed(%{"visit:source" => {:matches, "**facebook**"}})
test "member" do
"visit:country==FR|GB|DE"
|> assert_parsed(%{"visit:country" => {:member, ["FR", "GB", "DE"]}})
end
test "member + wildcard" do
"event:page==/blog**|/newsletter|/*/"
|> assert_parsed(%{"event:page" => {:matches, "/blog**|/newsletter|/*/"}})
end
test "combined with \";\"" do
"event:page==/blog**|/newsletter|/*/ ; visit:country==FR|GB|DE"
|> assert_parsed(%{
"event:page" => {:matches, "/blog**|/newsletter|/*/"},
"visit:country" => {:member, ["FR", "GB", "DE"]}
})
end
test "escaping pipe character" do
"utm_campaign==campaign \\| 1"
|> assert_parsed(%{"utm_campaign" => {:is, "campaign | 1"}})
end
test "escaping pipe character in member filter" do
"utm_campaign==campaign \\| 1|campaign \\| 2"
|> assert_parsed(%{"utm_campaign" => {:member, ["campaign | 1", "campaign | 2"]}})
end
test "keeps escape characters in member + wildcard filter" do
"event:page==/**\\|page|/other/page"
|> assert_parsed(%{"event:page" => {:matches, "/**\\|page|/other/page"}})
end
test "gracefully fails to parse garbage" do
"bfg10309\uff1cs1\ufe65s2\u02bas3\u02b9hjl10309"
|> assert_parsed(%{})
end
test "gracefully fails to parse garbage with double quotes" do
"\";print(md5(31337));$a=\""
|> assert_parsed(%{})
end
test "gracefully fails to parse garbage country code" do
"visit:country==AKSJSDFKJSS"
|> assert_parsed(%{})
end
test "gracefully fails to parse garbage country code (with pipes)" do
"visit:country==ET'||DBMS_PIPE.RECEIVE_MESSAGE(CHR(98)||CHR(98)||CHR(98),15)||'"
|> assert_parsed(%{})
end
end
end

6
test/plausible/stats/query_test.exs
View File

@ -188,14 +188,14 @@ defmodule Plausible.Stats.QueryTest do
filters = Jason.encode!(%{"goal" => "Signup"})
q = Query.from(site, %{"period" => "6mo", "filters" => filters})
assert q.filters["goal"] == "Signup"
assert q.filters["event:goal"] == {:is, {:event, "Signup"}}
end
test "parses source filter", %{site: site} do
filters = Jason.encode!(%{"source" => "Twitter"})
q = Query.from(site, %{"period" => "6mo", "filters" => filters})
assert q.filters["source"] == "Twitter"
assert q.filters["visit:source"] == {:is, "Twitter"}
end
test "allows prop filters when site owner is on a business plan", %{site: site, user: user} do
@ -203,7 +203,7 @@ defmodule Plausible.Stats.QueryTest do
filters = Jason.encode!(%{"props" => %{"author" => "!John Doe"}})
query = Query.from(site, %{"period" => "6mo", "filters" => filters})
assert Map.has_key?(query.filters, "props")
assert Map.has_key?(query.filters, "event:props:author")
end
test "drops prop filter when site owner is on a growth plan", %{site: site, user: user} do

235
test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
View File

@ -116,22 +116,28 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
}
end
test "validates that session metrics cannot be used with event:name filter", %{
conn: conn,
site: site
} do
conn =
get(conn, "/api/v1/stats/aggregate", %{
"site_id" => site.domain,
"period" => "30d",
"metrics" => "pageviews,visit_duration",
"filters" => "event:name==Signup"
})
for property <- ["event:name", "event:goal", "event:props:custom_prop"] do
test "validates that session metrics cannot be used with #{property} filter", %{
conn: conn,
site: site
} do
prop = unquote(property)
assert json_response(conn, 400) == %{
"error" =>
"Session metric `visit_duration` cannot be queried when using a filter on `event:name`."
}
if prop == "event:goal", do: insert(:goal, %{site: site, event_name: "some_value"})
conn =
get(conn, "/api/v1/stats/aggregate", %{
"site_id" => site.domain,
"period" => "30d",
"metrics" => "pageviews,visit_duration",
"filters" => "#{prop}==some_value"
})
assert json_response(conn, 400) == %{
"error" =>
"Session metric `visit_duration` cannot be queried when using a filter on `#{prop}`."
}
end
end
test "validates that views_per_visit cannot be used with event:page filter", %{
@ -400,6 +406,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
describe "filters" do
test "event:goal filter returns 400 when goal not configured", %{conn: conn, site: site} do
conn =
get(conn, "/api/v1/stats/aggregate", %{
"site_id" => site.domain,
"filters" => "event:goal==Register|Visit /register"
})
assert %{"error" => msg} = json_response(conn, 400)
assert msg =~ "The goal `Register` is not configured for this site. Find out how"
end
test "can filter by source", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
@ -864,6 +881,194 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
}
end
test "filtering by a custom event goal", %{conn: conn, site: site} do
populate_stats(site, [
build(:event,
name: "Signup",
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Signup",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event,
name: "Signup",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event,
name: "NotConfigured",
timestamp: ~N[2021-01-01 00:25:00]
)
])
insert(:goal, %{site: site, event_name: "Signup"})
conn =
get(conn, "/api/v1/stats/aggregate", %{
"site_id" => site.domain,
"period" => "day",
"date" => "2021-01-01",
"metrics" => "visitors,events",
"filters" => "event:goal==Signup"
})
assert json_response(conn, 200)["results"] == %{
"visitors" => %{"value" => 2},
"events" => %{"value" => 3}
}
end
test "filtering by a revenue goal", %{conn: conn, site: site} do
populate_stats(site, [
build(:event,
name: "Purchase",
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Purchase",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event,
name: "Purchase",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:25:00]
)
])
insert(:goal, site: site, currency: :USD, event_name: "Purchase")
conn =
get(conn, "/api/v1/stats/aggregate", %{
"site_id" => site.domain,
"period" => "day",
"date" => "2021-01-01",
"metrics" => "visitors,events",
"filters" => "event:goal==Purchase"
})
assert json_response(conn, 200)["results"] == %{
"visitors" => %{"value" => 2},
"events" => %{"value" => 3}
}
end
test "filtering by a simple pageview goal", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
pathname: "/register",
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/register",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview,
pathname: "/register",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview,
pathname: "/",
timestamp: ~N[2021-01-01 00:25:00]
)
])
insert(:goal, %{site: site, page_path: "/register"})
conn =
get(conn, "/api/v1/stats/aggregate", %{
"site_id" => site.domain,
"period" => "day",
"date" => "2021-01-01",
"metrics" => "visitors,pageviews",
"filters" => "event:goal==Visit /register"
})
assert json_response(conn, 200)["results"] == %{
"visitors" => %{"value" => 2},
"pageviews" => %{"value" => 3}
}
end
test "filtering by a wildcard pageview goal", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, pathname: "/blog/post-1"),
build(:pageview, pathname: "/blog/post-2", user_id: @user_id),
build(:pageview, pathname: "/blog", user_id: @user_id),
build(:pageview, pathname: "/")
])
insert(:goal, %{site: site, page_path: "/blog**"})
conn =
get(conn, "/api/v1/stats/aggregate", %{
"site_id" => site.domain,
"period" => "day",
"metrics" => "visitors,pageviews",
"filters" => "event:goal==Visit /blog**"
})
assert json_response(conn, 200)["results"] == %{
"visitors" => %{"value" => 2},
"pageviews" => %{"value" => 3}
}
end
test "filtering by multiple custom event goals", %{conn: conn, site: site} do
populate_stats(site, [
build(:event, name: "Signup"),
build(:event, name: "Purchase", user_id: @user_id),
build(:event, name: "Purchase", user_id: @user_id),
build(:pageview)
])
insert(:goal, %{site: site, event_name: "Signup"})
insert(:goal, %{site: site, event_name: "Purchase"})
conn =
get(conn, "/api/v1/stats/aggregate", %{
"site_id" => site.domain,
"period" => "day",
"metrics" => "visitors,events",
"filters" => "event:goal==Signup|Purchase"
})
assert json_response(conn, 200)["results"] == %{
"visitors" => %{"value" => 2},
"events" => %{"value" => 3}
}
end
test "filtering by multiple mixed goals", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, pathname: "/account/register"),
build(:pageview, pathname: "/register", user_id: @user_id),
build(:event, name: "Signup", user_id: @user_id),
build(:pageview)
])
insert(:goal, %{site: site, event_name: "Signup"})
insert(:goal, %{site: site, page_path: "/**register"})
conn =
get(conn, "/api/v1/stats/aggregate", %{
"site_id" => site.domain,
"period" => "day",
"metrics" => "visitors,events,pageviews",
"filters" => "event:goal==Signup|Visit /**register"
})
assert json_response(conn, 200)["results"] == %{
"visitors" => %{"value" => 2},
"events" => %{"value" => 3},
"pageviews" => %{"value" => 2}
}
end
test "combining filters", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,

142
test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
View File

@ -1179,28 +1179,22 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
describe "breakdown by event:goal" do
test "custom properties from custom events are returned", %{conn: conn, site: site} do
test "returns custom event goals and pageview goals", %{conn: conn, site: site} do
insert(:goal, %{site: site, event_name: "Purchase"})
insert(:goal, %{site: site, page_path: "/test"})
populate_stats(site, [
build(:pageview,
timestamp: ~N[2021-01-01 00:00:01],
pathname: "/test",
"meta.key": ["method"],
"meta.value": ["HTTP"]
pathname: "/test"
),
build(:event,
name: "Purchase",
timestamp: ~N[2021-01-01 00:00:03],
"meta.key": ["OS", "method"],
"meta.value": ["Linux", "HTTP"]
timestamp: ~N[2021-01-01 00:00:03]
),
build(:event,
name: "Purchase",
timestamp: ~N[2021-01-01 00:00:03],
"meta.key": ["OS"],
"meta.value": ["Linux"]
timestamp: ~N[2021-01-01 00:00:03]
)
])
@ -1213,24 +1207,66 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
})
assert [
%{
"goal" => "Purchase",
"props" => props,
"visitors" => 2
},
%{
"goal" => "Visit /test",
"props" => [],
"visitors" => 1
}
%{"goal" => "Purchase", "visitors" => 2},
%{"goal" => "Visit /test", "visitors" => 1}
] = json_response(conn, 200)["results"]
end
assert "method" in props
assert "OS" in props
test "returns pageview goals containing wildcards", %{conn: conn, site: site} do
insert(:goal, %{site: site, page_path: "/**/post"})
insert(:goal, %{site: site, page_path: "/blog**"})
populate_stats(site, [
build(:pageview, pathname: "/blog", user_id: @user_id),
build(:pageview, pathname: "/blog/post-1", user_id: @user_id),
build(:pageview, pathname: "/blog/post-2", user_id: @user_id),
build(:pageview, pathname: "/blog/something/post"),
build(:pageview, pathname: "/different/page/post")
])
conn =
get(conn, "/api/v1/stats/breakdown", %{
"site_id" => site.domain,
"metrics" => "visitors,pageviews",
"property" => "event:goal"
})
assert [
%{"goal" => "Visit /**/post", "visitors" => 2, "pageviews" => 2},
%{"goal" => "Visit /blog**", "visitors" => 2, "pageviews" => 4}
] = json_response(conn, 200)["results"]
end
test "does not return goals that are not configured for the site", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, pathname: "/register"),
build(:event, name: "Signup")
])
conn =
get(conn, "/api/v1/stats/breakdown", %{
"site_id" => site.domain,
"metrics" => "visitors,pageviews",
"property" => "event:goal"
})
assert [] = json_response(conn, 200)["results"]
end
end
describe "filtering" do
test "event:goal filter returns 400 when goal not configured", %{conn: conn, site: site} do
conn =
get(conn, "/api/v1/stats/breakdown", %{
"site_id" => site.domain,
"property" => "event:page",
"filters" => "event:goal==Register"
})
assert %{"error" => msg} = json_response(conn, 400)
assert msg =~ "The goal `Register` is not configured for this site. Find out how"
end
test "event:page filter for breakdown by session props", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
@ -1381,6 +1417,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "event:goal pageview filter for breakdown by visit source", %{conn: conn, site: site} do
insert(:goal, %{site: site, page_path: "/plausible.io"})
populate_stats(site, [
build(:pageview,
referrer_source: "Bing",
@ -1415,6 +1453,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "event:goal custom event filter for breakdown by visit source", %{conn: conn, site: site} do
insert(:goal, %{site: site, event_name: "Register"})
populate_stats(site, [
build(:pageview,
referrer_source: "Bing",
@ -1448,7 +1488,65 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
}
end
test "wildcard pageview goal filter for breakdown by event:page", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, pathname: "/en/register"),
build(:pageview, pathname: "/en/register", user_id: @user_id),
build(:pageview, pathname: "/en/register", user_id: @user_id),
build(:pageview, pathname: "/123/it/register"),
build(:pageview, pathname: "/should-not-appear")
])
insert(:goal, %{site: site, page_path: "/**register"})
conn =
get(conn, "/api/v1/stats/breakdown", %{
"site_id" => site.domain,
"period" => "day",
"metrics" => "visitors,pageviews",
"property" => "event:page",
"filters" => "event:goal==Visit /**register"
})
assert json_response(conn, 200) == %{
"results" => [
%{"page" => "/en/register", "visitors" => 2, "pageviews" => 3},
%{"page" => "/123/it/register", "visitors" => 1, "pageviews" => 1}
]
}
end
test "mixed multi-goal filter for breakdown by visit:country", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, country_code: "EE", pathname: "/en/register"),
build(:event, country_code: "EE", name: "Signup", pathname: "/en/register"),
build(:pageview, country_code: "US", pathname: "/123/it/register"),
build(:pageview, country_code: "US", pathname: "/different")
])
insert(:goal, %{site: site, page_path: "/**register"})
insert(:goal, %{site: site, event_name: "Signup"})
conn =
get(conn, "/api/v1/stats/breakdown", %{
"site_id" => site.domain,
"period" => "day",
"metrics" => "visitors,pageviews,events",
"property" => "visit:country",
"filters" => "event:goal==Signup|Visit /**register"
})
assert json_response(conn, 200) == %{
"results" => [
%{"country" => "EE", "visitors" => 2, "pageviews" => 1, "events" => 2},
%{"country" => "US", "visitors" => 1, "pageviews" => 1, "events" => 1}
]
}
end
test "event:goal custom event filter for breakdown by event page", %{conn: conn, site: site} do
insert(:goal, %{site: site, event_name: "Register"})
populate_stats(site, [
build(:event,
pathname: "/en/register",

192
test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs
View File

@ -466,6 +466,198 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
describe "filters" do
test "event:goal filter returns 400 when goal not configured", %{conn: conn, site: site} do
conn =
get(conn, "/api/v1/stats/aggregate", %{
"site_id" => site.domain,
"filters" => "event:goal==Visit /register**"
})
assert %{"error" => msg} = json_response(conn, 400)
assert msg =~
"The pageview goal for the pathname `/register**` is not configured for this site"
end
test "can filter by a custom event goal", %{conn: conn, site: site} do
populate_stats(site, [
build(:event,
name: "Signup",
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Signup",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event,
name: "Signup",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event,
name: "NotConfigured",
timestamp: ~N[2021-01-01 00:25:00]
)
])
insert(:goal, %{site: site, event_name: "Signup"})
conn =
get(conn, "/api/v1/stats/timeseries", %{
"site_id" => site.domain,
"period" => "month",
"date" => "2021-01-01",
"metrics" => "visitors,events",
"filters" => "event:goal==Signup"
})
res = json_response(conn, 200)
assert List.first(res["results"]) == %{
"date" => "2021-01-01",
"visitors" => 2,
"events" => 3
}
end
test "can filter by a simple pageview goal", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
pathname: "/register",
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/register",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview,
pathname: "/register",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview,
pathname: "/",
timestamp: ~N[2021-01-01 00:25:00]
)
])
insert(:goal, %{site: site, page_path: "/register"})
conn =
get(conn, "/api/v1/stats/timeseries", %{
"site_id" => site.domain,
"period" => "month",
"date" => "2021-01-01",
"metrics" => "visitors,pageviews",
"filters" => "event:goal==Visit /register"
})
res = json_response(conn, 200)
assert List.first(res["results"]) == %{
"date" => "2021-01-01",
"visitors" => 2,
"pageviews" => 3
}
end
test "can filter by a wildcard pageview goal", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, pathname: "/blog/post-1", timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview,
pathname: "/blog/post-2",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview, pathname: "/blog", user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00])
])
insert(:goal, %{site: site, page_path: "/blog**"})
conn =
get(conn, "/api/v1/stats/timeseries", %{
"site_id" => site.domain,
"period" => "month",
"date" => "2021-01-01",
"metrics" => "visitors,pageviews",
"filters" => "event:goal==Visit /blog**"
})
res = json_response(conn, 200)
assert List.first(res["results"]) == %{
"date" => "2021-01-01",
"visitors" => 2,
"pageviews" => 3
}
end
test "can filter by multiple custom event goals", %{conn: conn, site: site} do
populate_stats(site, [
build(:event, name: "Signup", timestamp: ~N[2021-01-01 00:25:00]),
build(:event, name: "Purchase", user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]),
build(:event, name: "Purchase", user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview, timestamp: ~N[2021-01-01 00:25:00])
])
insert(:goal, %{site: site, event_name: "Signup"})
insert(:goal, %{site: site, event_name: "Purchase"})
conn =
get(conn, "/api/v1/stats/timeseries", %{
"site_id" => site.domain,
"period" => "month",
"date" => "2021-01-01",
"metrics" => "visitors,events",
"filters" => "event:goal==Signup|Purchase"
})
res = json_response(conn, 200)
assert List.first(res["results"]) == %{
"date" => "2021-01-01",
"visitors" => 2,
"events" => 3
}
end
test "can filter by multiple mixed goals", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, pathname: "/account/register", timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview,
pathname: "/register",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event, name: "Signup", user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview, timestamp: ~N[2021-01-01 00:25:00])
])
insert(:goal, %{site: site, event_name: "Signup"})
insert(:goal, %{site: site, page_path: "/**register"})
conn =
get(conn, "/api/v1/stats/timeseries", %{
"site_id" => site.domain,
"period" => "month",
"date" => "2021-01-01",
"metrics" => "visitors,events,pageviews",
"filters" => "event:goal==Signup|Visit /**register"
})
res = json_response(conn, 200)
assert List.first(res["results"]) == %{
"date" => "2021-01-01",
"visitors" => 2,
"events" => 3,
"pageviews" => 2
}
end
test "can filter by source", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,