plausible/analytics
1
1
Fork 0
mirror of https://github.com/plausible/analytics.git synced 2024-11-22 10:43:38 +03:00
Code Issues Packages Projects Releases Wiki Activity

APIv2: Regex operations, consistent operators (#4488)

Browse Source 
* Rename matches/does_not_match filters internally

These have never been exposed to the frontend/user directly, only via
APIv1 filtering syntax. As such we are free to rename these without
breaking things

* Rename function arguments for consistency, simplify

* Add support for `match`/`not_match` operators for query apiv2

These match the string against a regular expression, as defined in
https://github.com/google/re2/wiki/Syntax

* not_match -> match_not

* does_not_contain -> contains_not

Note that for backwards compatibility:
- Browser handles does_not_contain in URL
- Backend will handle does_not_contain in queries for a day where we will remove it for better autocompletion

* not_matches_wildcard -> matches_wildcard_not

* prettier

* match -> matches

* Fix and test fix for matches_wildcard against prop when prop is missing

* Custom properties support for matches/matches_not

* Restore contains_not

* Test contains and contains_not behavior for custom properties
This commit is contained in:
Karl-Aksel Puulmann 2024-09-09 10:05:24 +03:00 committed by GitHub
parent 67d7c6522c
commit 604dde99fd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
23 changed files with 298 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -11,7 +11,7 @@ All notable changes to this project will be documented in this file.
- Add a search functionality in all Details views
- Icons for browsers plausible/analytics#4239
- Automatic custom property selection in the dashboard Properties report
- Add `does_not_contain` filter support to dashboard
- Add `contains_not` filter support to dashboard
- Traffic drop notifications plausible/analytics#4300
- Add search and pagination functionality into Google Keywords > Details modal
- ClickHouse system.query_log table log_comment column now contains information about source of queries. Useful for debugging

2
assets/js/dashboard/components/filter-operator-selector.js
View File

@ -63,7 +63,7 @@ export default function FilterOperatorSelector(props) {
{renderTypeItem(FILTER_OPERATIONS.is, true)}
{renderTypeItem(FILTER_OPERATIONS.isNot, supportsIsNot(filterName))}
{renderTypeItem(FILTER_OPERATIONS.contains, isFreeChoiceFilter(filterName))}
{renderTypeItem(FILTER_OPERATIONS.does_not_contain, isFreeChoiceFilter(filterName) && supportsIsNot(filterName))}
{renderTypeItem(FILTER_OPERATIONS.contains_not, isFreeChoiceFilter(filterName) && supportsIsNot(filterName))}
</div>
</Menu.Items>
</Transition>

9
assets/js/dashboard/query-context.tsx
View File

@ -21,7 +21,12 @@ import {
QueryPeriod,
useSaveTimePreferencesToStorage
} from './query-time-periods'
import { Filter, FilterClauseLabels, queryDefaultValue } from './query'
import {
Filter,
FilterClauseLabels,
queryDefaultValue,
postProcessFilters
} from './query'
const queryContextDefaultValue = {
query: queryDefaultValue,
@ -99,7 +104,7 @@ export default function QueryContextProvider({
? (with_imported as boolean)
: defaultValues.with_imported,
filters: Array.isArray(filters)
? (filters as Filter[])
? postProcessFilters(filters as Filter[])
: defaultValues.filters,
labels: (labels as FilterClauseLabels) || defaultValues.labels
}

11
assets/js/dashboard/query.ts
View File

@ -11,6 +11,7 @@ import {
isAfter
} from './util/date'
import {
FILTER_OPERATIONS,
getFiltersByKeyPrefix,
parseLegacyFilter,
parseLegacyPropsFilter
@ -84,6 +85,16 @@ const LEGACY_URL_PARAMETERS = {
exit_page: null
}
export function postProcessFilters(filters: Array<Filter>): Array<Filter> {
return filters.map(([operation, dimension, clauses]) => {
// Rename old name of the operation
if (operation === 'does_not_contain') {
operation = FILTER_OPERATIONS.contains_not
}
return [operation, dimension, clauses]
})
}
// Called once when dashboard is loaded load. Checks whether old filter style is used and if so,
// updates the filters and updates location
export function filtersBackwardsCompatibilityRedirect(

2
assets/js/dashboard/stats/modals/filter-modal-props-row.js
View File

@ -35,7 +35,7 @@ export default function FilterModalPropsRow({
}
function fetchPropValueOptions(input) {
if ([FILTER_OPERATIONS.contains, FILTER_OPERATIONS.does_not_contain].includes(operation) || propKey == "") {
if ([FILTER_OPERATIONS.contains, FILTER_OPERATIONS.contains_not].includes(operation) || propKey == "") {
return Promise.resolve([])
}
return fetchSuggestions(apiPath(site, `/suggestions/prop_value`), query, input, [

2
assets/js/dashboard/stats/modals/filter-modal-row.js
View File

@ -34,7 +34,7 @@ export default function FilterModalRow({
}
function fetchOptions(input) {
if ([FILTER_OPERATIONS.contains, FILTER_OPERATIONS.does_not_contain].includes(operation)) {
if ([FILTER_OPERATIONS.contains, FILTER_OPERATIONS.contains_not].includes(operation)) {
return Promise.resolve([])
}

4
assets/js/dashboard/util/filters.js
View File

@ -28,14 +28,14 @@ export const FILTER_OPERATIONS = {
is: 'is',
isNot: 'is_not',
contains: 'contains',
does_not_contain: 'does_not_contain'
contains_not: 'contains_not'
};
export const FILTER_OPERATIONS_DISPLAY_NAMES = {
[FILTER_OPERATIONS.is]: 'is',
[FILTER_OPERATIONS.isNot]: 'is not',
[FILTER_OPERATIONS.contains]: 'contains',
[FILTER_OPERATIONS.does_not_contain]: 'does not contain'
[FILTER_OPERATIONS.contains_not]: 'does not contain'
}
const OPERATION_PREFIX = {

2
lib/plausible/google/search_console/filters.ex
View File

@ -31,7 +31,7 @@ defmodule Plausible.Google.SearchConsole.Filters do
%{dimension: "page", operator: "includingRegex", expression: expression}
end
defp transform_filter(property, [:matches, "visit:entry_page", pages])
defp transform_filter(property, [:matches_wildcard, "visit:entry_page", pages])
when is_list(pages) do
expression =
Enum.map_join(pages, "|", fn page -> page_regex(property_url(property, page)) end)

2
lib/plausible/stats/filters/filters.ex
View File

@ -60,7 +60,7 @@ defmodule Plausible.Stats.Filters do
### Examples:
iex> Filters.parse("{\\"page\\":\\"/blog/**\\"}")
[[:matches, "event:page", ["/blog/**"]]]
[[:matches_wildcard, "event:page", ["/blog/**"]]]
iex> Filters.parse("visit:browser!=Chrome")
[[:is_not, "visit:browser", ["Chrome"]]]

12
lib/plausible/stats/filters/legacy_dashboard_filter_parser.ex
View File

@ -35,23 +35,23 @@ defmodule Plausible.Stats.Filters.LegacyDashboardFilterParser do
cond do
is_negated && is_wildcard && is_list ->
[:does_not_match, key, val]
[:matches_wildcard_not, key, val]
# TODO
is_negated && is_contains && is_list ->
[:does_not_match, key, Enum.map(val, &"**#{&1}**")]
[:matches_wildcard_not, key, Enum.map(val, &"**#{&1}**")]
is_wildcard && is_list ->
[:matches, key, val]
[:matches_wildcard, key, val]
is_negated && is_wildcard ->
[:does_not_match, key, [val]]
[:matches_wildcard_not, key, [val]]
is_negated && is_list ->
[:is_not, key, val]
is_negated && is_contains ->
[:does_not_match, key, ["**" <> val <> "**"]]
[:matches_wildcard_not, key, ["**" <> val <> "**"]]
is_negated ->
[:is_not, key, [val]]
@ -66,7 +66,7 @@ defmodule Plausible.Stats.Filters.LegacyDashboardFilterParser do
[:contains, key, [val]]
is_wildcard ->
[:matches, key, [val]]
[:matches_wildcard, key, [val]]
true ->
[:is, key, [val]]

19
lib/plausible/stats/filters/query_parser.ex
View File

@ -74,9 +74,13 @@ defmodule Plausible.Stats.Filters.QueryParser do
defp parse_operator(["is" | _rest]), do: {:ok, :is}
defp parse_operator(["is_not" | _rest]), do: {:ok, :is_not}
defp parse_operator(["matches" | _rest]), do: {:ok, :matches}
defp parse_operator(["does_not_match" | _rest]), do: {:ok, :does_not_match}
defp parse_operator(["matches_not" | _rest]), do: {:ok, :matches_not}
defp parse_operator(["matches_wildcard" | _rest]), do: {:ok, :matches_wildcard}
defp parse_operator(["matches_wildcard_not" | _rest]), do: {:ok, :matches_wildcard_not}
defp parse_operator(["contains" | _rest]), do: {:ok, :contains}
defp parse_operator(["does_not_contain" | _rest]), do: {:ok, :does_not_contain}
defp parse_operator(["contains_not" | _rest]), do: {:ok, :contains_not}
# :TODO: Remove this once frontend support is gone
defp parse_operator(["does_not_contain" | _rest]), do: {:ok, :contains_not}
defp parse_operator(["not" | _rest]), do: {:ok, :not}
defp parse_operator(["and" | _rest]), do: {:ok, :and}
defp parse_operator(["or" | _rest]), do: {:ok, :or}
@ -96,7 +100,16 @@ defmodule Plausible.Stats.Filters.QueryParser do
defp parse_filter_key(filter), do: {:error, "Invalid filter '#{i(filter)}'."}
defp parse_filter_rest(operator, filter)
when operator in [:is, :is_not, :matches, :does_not_match, :contains, :does_not_contain],
when operator in [
:is,
:is_not,
:matches,
:matches_not,
:matches_wildcard,
:matches_wildcard_not,
:contains,
:contains_not
],
do: parse_clauses_list(filter)
defp parse_filter_rest(operator, _filter)

4
lib/plausible/stats/filters/stats_api_filter_parser.ex
View File

@ -27,8 +27,8 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser do
final_value = remove_escape_chars(raw_value)
cond do
is_wildcard? && is_negated? -> [:does_not_match, key, [raw_value]]
is_wildcard? -> [:matches, key, [raw_value]]
is_wildcard? && is_negated? -> [:matches_wildcard_not, key, [raw_value]]
is_wildcard? -> [:matches_wildcard, key, [raw_value]]
is_list? -> [:is, key, parse_member_list(raw_value)]
is_negated? -> [:is_not, key, [final_value]]
true -> [:is, key, [final_value]]

8
lib/plausible/stats/json_schema.ex
View File

@ -17,12 +17,12 @@ defmodule Plausible.Stats.JSONSchema do
@internal_query_schema @raw_public_schema
# Add overrides for things allowed in the internal API
|> JSONPointer.add!(
"#/definitions/filter_entry/oneOf/0/items/0/enum/0",
"matches"
"#/definitions/filter_entry/oneOf/0/items/0/oneOf/0/enum/0",
"matches_wildcard"
)
|> JSONPointer.add!(
"#/definitions/filter_entry/oneOf/0/items/0/enum/0",
"does_not_match"
"#/definitions/filter_entry/oneOf/0/items/0/oneOf/0/enum/0",
"matches_wildcard_not"
)
|> JSONPointer.add!("#/definitions/metric/oneOf/0", %{
"const" => "time_on_page"

72
lib/plausible/stats/sql/where_builder.ex
View File

@ -156,59 +156,67 @@ defmodule Plausible.Stats.SQL.WhereBuilder do
false
end
defp filter_custom_prop(prop_name, column_name, [:is, _, values]) do
none_value_included = Enum.member?(values, "(none)")
defp filter_custom_prop(prop_name, column_name, [:is, _, clauses]) do
none_value_included = Enum.member?(clauses, "(none)")
dynamic(
[t],
(has_key(t, column_name, ^prop_name) and get_by_key(t, column_name, ^prop_name) in ^values) or
(has_key(t, column_name, ^prop_name) and get_by_key(t, column_name, ^prop_name) in ^clauses) or
(^none_value_included and not has_key(t, column_name, ^prop_name))
)
end
defp filter_custom_prop(prop_name, column_name, [:is_not, _, values]) do
none_value_included = Enum.member?(values, "(none)")
defp filter_custom_prop(prop_name, column_name, [:is_not, _, clauses]) do
none_value_included = Enum.member?(clauses, "(none)")
dynamic(
[t],
(has_key(t, column_name, ^prop_name) and
get_by_key(t, column_name, ^prop_name) not in ^values) or
get_by_key(t, column_name, ^prop_name) not in ^clauses) or
(^none_value_included and
has_key(t, column_name, ^prop_name) and
get_by_key(t, column_name, ^prop_name) not in ^values) or
get_by_key(t, column_name, ^prop_name) not in ^clauses) or
(not (^none_value_included) and not has_key(t, column_name, ^prop_name))
)
end
defp filter_custom_prop(prop_name, column_name, [:matches, _, clauses]) do
defp filter_custom_prop(prop_name, column_name, [:matches_wildcard, dimension, clauses]) do
regexes = Enum.map(clauses, &page_regex/1)
filter_custom_prop(prop_name, column_name, [:matches, dimension, regexes])
end
defp filter_custom_prop(prop_name, column_name, [:matches_wildcard_not, dimension, clauses]) do
regexes = Enum.map(clauses, &page_regex/1)
filter_custom_prop(prop_name, column_name, [:matches_not, dimension, regexes])
end
defp filter_custom_prop(prop_name, column_name, [:matches, _dimension, clauses]) do
dynamic(
[t],
has_key(t, column_name, ^prop_name) and
fragment(
"arrayExists(k -> match(?, k), ?)",
get_by_key(t, column_name, ^prop_name),
^regexes
^clauses
)
)
end
defp filter_custom_prop(prop_name, column_name, [:does_not_match, _, clauses]) do
regexes = Enum.map(clauses, &page_regex/1)
defp filter_custom_prop(prop_name, column_name, [:matches_not, _dimension, clauses]) do
dynamic(
[t],
has_key(t, column_name, ^prop_name) and
fragment(
"not(arrayExists(k -> match(?, k), ?))",
get_by_key(t, column_name, ^prop_name),
^regexes
^clauses
)
)
end
defp filter_custom_prop(prop_name, column_name, [:contains, _, clauses]) do
defp filter_custom_prop(prop_name, column_name, [:contains, _dimension, clauses]) do
dynamic(
[t],
has_key(t, column_name, ^prop_name) and
@ -220,7 +228,7 @@ defmodule Plausible.Stats.SQL.WhereBuilder do
)
end
defp filter_custom_prop(prop_name, column_name, [:does_not_contain, _, clauses]) do
defp filter_custom_prop(prop_name, column_name, [:contains_not, _dimension, clauses]) do
dynamic(
[t],
has_key(t, column_name, ^prop_name) and
@ -232,7 +240,7 @@ defmodule Plausible.Stats.SQL.WhereBuilder do
)
end
defp filter_field(db_field, [:matches, _key, glob_exprs]) do
defp filter_field(db_field, [:matches_wildcard, _dimension, glob_exprs]) do
page_regexes = Enum.map(glob_exprs, &page_regex/1)
dynamic(
@ -241,34 +249,36 @@ defmodule Plausible.Stats.SQL.WhereBuilder do
)
end
defp filter_field(db_field, [:does_not_match, _key, glob_exprs]) do
page_regexes = Enum.map(glob_exprs, &page_regex/1)
dynamic(
[x],
fragment("not(multiMatchAny(?, ?))", type(field(x, ^db_field), :string), ^page_regexes)
)
defp filter_field(db_field, [:matches_wildcard_not, dimension, clauses]) do
dynamic([], not (^filter_field(db_field, [:matches_wildcard, dimension, clauses])))
end
defp filter_field(db_field, [:contains, _key, values]) do
defp filter_field(db_field, [:contains, _dimension, values]) do
dynamic([x], fragment("multiSearchAny(?, ?)", type(field(x, ^db_field), :string), ^values))
end
defp filter_field(db_field, [:does_not_contain, _key, values]) do
defp filter_field(db_field, [:contains_not, dimension, clauses]) do
dynamic([], not (^filter_field(db_field, [:contains, dimension, clauses])))
end
defp filter_field(db_field, [:matches, _dimension, clauses]) do
dynamic(
[x],
fragment("not(multiSearchAny(?, ?))", type(field(x, ^db_field), :string), ^values)
fragment("multiMatchAny(?, ?)", type(field(x, ^db_field), :string), ^clauses)
)
end
defp filter_field(db_field, [:is, _key, list]) do
list = Enum.map(list, &db_field_val(db_field, &1))
defp filter_field(db_field, [:matches_not, dimension, clauses]) do
dynamic([], not (^filter_field(db_field, [:matches, dimension, clauses])))
end
defp filter_field(db_field, [:is, _dimension, clauses]) do
list = Enum.map(clauses, &db_field_val(db_field, &1))
dynamic([x], field(x, ^db_field) in ^list)
end
defp filter_field(db_field, [:is_not, _key, list]) do
list = Enum.map(list, &db_field_val(db_field, &1))
dynamic([x], field(x, ^db_field) not in ^list)
defp filter_field(db_field, [:is_not, dimension, clauses]) do
dynamic([], not (^filter_field(db_field, [:is, dimension, clauses])))
end
@no_ref "Direct / None"

19
priv/json-schemas/query-api-schema.json
View File

@ -243,10 +243,21 @@
"type": "array",
"items": [
{
"type": "string",
"enum": [
"is_not",
"does_not_contain"
"oneOf": [
{
"type": "string",
"enum": [
"is_not",
"contains_not",
"matches",
"matches_not"
]
},
{
"const": "does_not_contain",
"deprecationMessage": "Legacy support, will be removed next week",
"deprecated": true
}
],
"description": "filter operation"
},

12
test/plausible/google/search_console/filters_test.exs
View File

@ -22,9 +22,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
]
end
test "transforms matches page filter" do
test "transforms matches_wildcard page filter" do
filters = [
[:matches, "visit:entry_page", ["*page*"]]
[:matches_wildcard, "visit:entry_page", ["*page*"]]
]
{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters, "")
@ -64,7 +64,7 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
test "transforms matches multiple page filter" do
filters = [
[:matches, "visit:entry_page", ["/pageA*", "/pageB*"]]
[:matches_wildcard, "visit:entry_page", ["/pageA*", "/pageB*"]]
]
{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters, "")
@ -84,7 +84,7 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
test "transforms event:page exactly like visit:entry_page" do
filters = [
[:matches, "event:page", ["/pageA*", "/pageB*"]]
[:matches_wildcard, "event:page", ["/pageA*", "/pageB*"]]
]
{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters, "")
@ -165,7 +165,7 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
test "filters can be combined" do
filters = [
[:is, "visit:country", ["EE", "PL"]],
[:matches, "visit:entry_page", ["*web-analytics*"]],
[:matches_wildcard, "visit:entry_page", ["*web-analytics*"]],
[:is, "visit:screen", ["Desktop"]]
]
@ -188,7 +188,7 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
test "when unsupported filter is included the whole set becomes invalid" do
filters = [
[:matches, "visit:entry_page", "*web-analytics*"],
[:matches_wildcard, "visit:entry_page", "*web-analytics*"],
[:is, "visit:screen", "Desktop"],
[:member, "visit:country", ["EE", "PL"]],
[:is, "visit:utm_medium", "facebook"]

10
test/plausible/stats/filters_test.exs
View File

@ -27,12 +27,12 @@ defmodule Plausible.Stats.FiltersTest do
test "wildcard" do
"event:page==/blog/post-*"
|> assert_parsed([[:matches, "event:page", ["/blog/post-*"]]])
|> assert_parsed([[:matches_wildcard, "event:page", ["/blog/post-*"]]])
end
test "negative wildcard" do
"event:page!=/blog/post-*"
|> assert_parsed([[:does_not_match, "event:page", ["/blog/post-*"]]])
|> assert_parsed([[:matches_wildcard_not, "event:page", ["/blog/post-*"]]])
end
test "custom event goal" do
@ -52,13 +52,13 @@ defmodule Plausible.Stats.FiltersTest do
test "member + wildcard" do
"event:page==/blog**|/newsletter|/*/"
|> assert_parsed([[:matches, "event:page", ["/blog**|/newsletter|/*/"]]])
|> assert_parsed([[:matches_wildcard, "event:page", ["/blog**|/newsletter|/*/"]]])
end
test "combined with \";\"" do
"event:page==/blog**|/newsletter|/*/ ; visit:country==FR|GB|DE"
|> assert_parsed([
[:matches, "event:page", ["/blog**|/newsletter|/*/"]],
[:matches_wildcard, "event:page", ["/blog**|/newsletter|/*/"]],
[:is, "visit:country", ["FR", "GB", "DE"]]
])
end
@ -75,7 +75,7 @@ defmodule Plausible.Stats.FiltersTest do
test "keeps escape characters in is + wildcard filter" do
"event:page==/**\\|page|/other/page"
|> assert_parsed([[:matches, "event:page", ["/**\\|page|/other/page"]]])
|> assert_parsed([[:matches_wildcard, "event:page", ["/**\\|page|/other/page"]]])
end
test "gracefully fails to parse garbage" do

12
test/plausible/stats/legacy_dashboard_filter_parser_test.exs
View File

@ -115,15 +115,15 @@ defmodule Plausible.Stats.Legacy.DashboardFilterParserTest do
end
end
describe "matches filter type" do
describe "matches_wildcard filter type" do
test "parses matches filter type" do
%{"page" => "/|/blog**"}
|> assert_parsed([[:matches, "event:page", ["/", "/blog**"]]])
|> assert_parsed([[:matches_wildcard, "event:page", ["/", "/blog**"]]])
end
test "parses not_matches filter type" do
%{"page" => "!/|/blog**"}
|> assert_parsed([[:does_not_match, "event:page", ["/", "/blog**"]]])
|> assert_parsed([[:matches_wildcard_not, "event:page", ["/", "/blog**"]]])
end
test "single matches" do
@ -133,7 +133,7 @@ defmodule Plausible.Stats.Legacy.DashboardFilterParserTest do
test "negated matches" do
%{"page" => "!~articles"}
|> assert_parsed([[:does_not_match, "event:page", ["**articles**"]]])
|> assert_parsed([[:matches_wildcard_not, "event:page", ["**articles**"]]])
end
test "matches member" do
@ -143,7 +143,7 @@ defmodule Plausible.Stats.Legacy.DashboardFilterParserTest do
test "not matches member" do
%{"page" => "!~articles|blog"}
|> assert_parsed([[:does_not_match, "event:page", ["**articles**", "**blog**"]]])
|> assert_parsed([[:matches_wildcard_not, "event:page", ["**articles**", "**blog**"]]])
end
test "other filters default to `is` even when wildcard is present" do
@ -153,7 +153,7 @@ defmodule Plausible.Stats.Legacy.DashboardFilterParserTest do
test "can be used with `page` filter" do
%{"page" => "!/blog/post-*"}
|> assert_parsed([[:does_not_match, "event:page", ["/blog/post-*"]]])
|> assert_parsed([[:matches_wildcard_not, "event:page", ["/blog/post-*"]]])
end
test "other filters default to is_not even when wildcard is present" do

8
test/plausible/stats/query_optimizer_test.exs
View File

@ -127,16 +127,16 @@ defmodule Plausible.Stats.QueryOptimizerTest do
date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]),
filters: [
[:is, "event:hostname", ["example.com"]],
[:matches, "event:hostname", ["*.com"]]
[:matches_wildcard, "event:hostname", ["*.com"]]
],
dimensions: ["visit:referrer", "visit:exit_page"]
}).filters == [
[:is, "event:hostname", ["example.com"]],
[:matches, "event:hostname", ["*.com"]],
[:matches_wildcard, "event:hostname", ["*.com"]],
[:is, "visit:entry_page_hostname", ["example.com"]],
[:matches, "visit:entry_page_hostname", ["*.com"]],
[:matches_wildcard, "visit:entry_page_hostname", ["*.com"]],
[:is, "visit:exit_page_hostname", ["example.com"]],
[:matches, "visit:exit_page_hostname", ["*.com"]]
[:matches_wildcard, "visit:exit_page_hostname", ["*.com"]]
]
end

13
test/plausible/stats/query_parser_test.exs
View File

@ -167,7 +167,16 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
end
describe "filters validation" do
for operation <- [:is, :is_not, :matches, :does_not_match, :contains, :does_not_contain] do
for operation <- [
:is,
:is_not,
:matches_wildcard,
:matches_wildcard_not,
:matches,
:matches_not,
:contains,
:contains_not
] do
test "#{operation} filter", %{site: site} do
%{
"site_id" => site.domain,
@ -212,7 +221,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
end
end
for operation <- [:matches, :does_not_match] do
for operation <- [:matches_wildcard, :matches_wildcard_not] do
test "#{operation} is not a valid filter operation in public API", %{site: site} do
%{
"site_id" => site.domain,

15
test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
View File

@ -1852,7 +1852,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
"site_id" => site.domain,
"metrics" => "visitors",
"filters" => [
["does_not_contain", "event:page", ["/en*"]]
["contains_not", "event:page", ["/en*"]]
]
})
@ -1884,14 +1884,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
"site_id" => site.domain,
"metrics" => "visitors",
"filters" => [
["matches", "event:props:tier", ["small*"]]
["matches_wildcard", "event:props:tier", ["small*"]]
]
})
assert json_response(conn, 200)["results"] == %{"visitors" => %{"value" => 3}}
end
test "does_not_match custom event property", %{conn: conn, site: site} do
test "not matches_wildcard custom event property", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
"meta.key": ["tier"],
@ -1908,7 +1908,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
build(:pageview,
"meta.key": ["tier"],
"meta.value": ["small-2"]
)
),
build(:pageview)
])
conn =
@ -1916,7 +1917,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
"site_id" => site.domain,
"metrics" => "visitors",
"filters" => [
["does_not_match", "event:props:tier", ["small*"]]
["matches_wildcard_not", "event:props:tier", ["small*"]]
]
})
@ -1955,7 +1956,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
assert json_response(conn, 200)["results"] == %{"visitors" => %{"value" => 3}}
end
test "does_not_contain custom event property", %{conn: conn, site: site} do
test "contains_not custom event property", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
"meta.key": ["tier"],
@ -1980,7 +1981,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
"site_id" => site.domain,
"metrics" => "visitors",
"filters" => [
["does_not_contain", "event:props:tier", ["small"]]
["contains_not", "event:props:tier", ["small"]]
]
})

148
test/plausible_web/controllers/api/external_stats_controller/query_test.exs
View File

@ -752,7 +752,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
assert json_response(conn, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}]
end
test "does_not_contain page filter", %{conn: conn, site: site} do
test "contains_not page filter", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, pathname: "/en/page1"),
build(:pageview, pathname: "/en/page2"),
@ -765,7 +765,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
"date_range" => "all",
"metrics" => ["visitors"],
"filters" => [
["does_not_contain", "event:page", ["/en/"]]
["contains_not", "event:page", ["/en/"]]
]
})
@ -848,6 +848,150 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
assert json_response(conn, 200)["results"] == [%{"metrics" => [3], "dimensions" => []}]
end
test "`matches` operator", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, pathname: "/user/1234"),
build(:pageview, pathname: "/user/789/contributions"),
build(:pageview, pathname: "/blog/user/1234"),
build(:pageview, pathname: "/user/ef/contributions"),
build(:pageview, pathname: "/other/path")
])
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"date_range" => "all",
"metrics" => ["visitors"],
"filters" => [
["matches", "event:page", ["^/user/[0-9]+"]]
]
})
assert json_response(conn, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}]
end
test "`matches_not` operator", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, pathname: "/user/1234"),
build(:pageview, pathname: "/user/789/contributions"),
build(:pageview, pathname: "/blog/user/1234"),
build(:pageview, pathname: "/user/ef/contributions"),
build(:pageview, pathname: "/other/path")
])
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"date_range" => "all",
"metrics" => ["visitors"],
"filters" => [
["matches_not", "event:page", ["^/user/[0-9]+"]]
]
})
assert json_response(conn, 200)["results"] == [%{"metrics" => [3], "dimensions" => []}]
end
test "`contains` and `contains_not` operator with custom properties", %{
conn: conn,
site: site
} do
populate_stats(site, [
build(:pageview,
"meta.key": ["tier", "value"],
"meta.value": ["large-1", "ax"]
),
build(:pageview,
"meta.key": ["tier", "value"],
"meta.value": ["small-1", "bx"]
),
build(:pageview,
"meta.key": ["tier", "value"],
"meta.value": ["small-1", "ax"]
),
build(:pageview,
"meta.key": ["tier", "value"],
"meta.value": ["small-2", "bx"]
),
build(:pageview,
"meta.key": ["tier", "value"],
"meta.value": ["small-2", "cx"]
),
build(:pageview,
"meta.key": ["tier"],
"meta.value": ["small-3"]
),
build(:pageview,
"meta.key": ["value"],
"meta.value": ["ax"]
),
build(:pageview)
])
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"date_range" => "all",
"metrics" => ["visitors"],
"filters" => [
["contains", "event:props:tier", ["small"]],
["contains_not", "event:props:value", ["b", "c"]]
]
})
assert json_response(conn, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}]
end
test "`matches` and `matches_not` operator with custom properties", %{
conn: conn,
site: site
} do
populate_stats(site, [
build(:pageview,
"meta.key": ["tier", "value"],
"meta.value": ["large-1", "a"]
),
build(:pageview,
"meta.key": ["tier", "value"],
"meta.value": ["small-1", "b"]
),
build(:pageview,
"meta.key": ["tier", "value"],
"meta.value": ["small-1", "a"]
),
build(:pageview,
"meta.key": ["tier", "value"],
"meta.value": ["small-2", "b"]
),
build(:pageview,
"meta.key": ["tier", "value"],
"meta.value": ["small-2", "c"]
),
build(:pageview,
"meta.key": ["tier"],
"meta.value": ["small-3"]
),
build(:pageview,
"meta.key": ["value"],
"meta.value": ["a"]
),
build(:pageview)
])
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"date_range" => "all",
"metrics" => ["visitors"],
"filters" => [
["matches", "event:props:tier", ["small.+"]],
["matches_not", "event:props:value", ["b|c"]]
]
})
assert json_response(conn, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}]
end
test "handles filtering by visit:country", %{
conn: conn,
site: site

4
test/plausible_web/controllers/api/stats_controller/pages_test.exs
View File

@ -106,7 +106,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
]
end
test "returns top pages with :matches filter on custom pageview props", %{
test "returns top pages with :matches_wildcard filter on custom pageview props", %{
conn: conn,
site: site
} do
@ -1328,7 +1328,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
]
end
test "filter by :matches page with imported data", %{conn: conn, site: site} do
test "filter by :matches_wildcard page with imported data", %{conn: conn, site: site} do
site_import = insert(:site_import, site: site)
populate_stats(site, site_import.id, [