plausible/analytics
1
1
Fork 0
mirror of https://github.com/plausible/analytics.git synced 2024-08-17 05:40:24 +03:00
Code Issues Packages Projects Releases Wiki Activity

does_not_contain, new filters wildcard matching (#4228)

Browse Source 
* Make goals accept wildcards in :is queries

* Add support for contains/does_not_contains in the backend

* Support all operations for event custom properties

* Support does_not_contain on the frontend

* Changelog entry

* Render filter operations nicely for does not contain

Found 3 extra pixels for operation dropdown

* Remove multiple_filters feature flag
This commit is contained in:
Karl-Aksel Puulmann 2024-06-21 11:32:05 +03:00 committed by GitHub
parent ed8abf426b
commit 284b4ebbf6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 355 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -12,6 +12,7 @@ All notable changes to this project will be documented in this file.
### Changed
- Increase hourly request limit for API keys in CE from 600 to 1000000 (practically removing the limit) plausible/analytics#4200
- Add `does_not_contain` filter support to dashboard
### Fixed

15
assets/js/dashboard/components/filter-operator-selector.js
View File

@ -1,6 +1,6 @@
import React, { Fragment } from "react";
import { FILTER_OPERATIONS } from "../util/filters";
import { FILTER_OPERATIONS, FILTER_OPERATIONS_DISPLAY_NAMES } from "../util/filters";
import { Menu, Transition } from "@headlessui/react";
import { ChevronDownIcon } from '@heroicons/react/20/solid'
import { isFreeChoiceFilter, supportsIsNot } from "../util/filters";
@ -9,20 +9,20 @@ import classNames from "classnames";
export default function FilterOperatorSelector(props) {
const filterName = props.forFilter
function renderTypeItem(type, shouldDisplay) {
function renderTypeItem(operation, shouldDisplay) {
return (
shouldDisplay && (
<Menu.Item>
{({ active }) => (
<span
onClick={() => props.onSelect(type)}
onClick={() => props.onSelect(operation)}
className={classNames("cursor-pointer block px-4 py-2 text-sm", {
"bg-gray-100 dark:bg-gray-900 text-gray-900 dark:text-gray-100": active,
"text-gray-700 dark:text-gray-200": !active
}
)}
>
{type}
{FILTER_OPERATIONS_DISPLAY_NAMES[operation]}
</span>
)}
</Menu.Item>
@ -40,8 +40,8 @@ export default function FilterOperatorSelector(props) {
{({ open }) => (
<>
<div className="w-full">
<Menu.Button className="inline-flex justify-between items-center w-full rounded-md border border-gray-300 dark:border-gray-500 shadow-sm px-4 py-2 bg-white dark:bg-gray-800 text-sm text-gray-700 dark:text-gray-200 hover:bg-gray-50 dark:hover:bg-gray-850 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-offset-gray-100 dark:focus:ring-offset-gray-900 focus:ring-indigo-500">
{props.selectedType}
<Menu.Button className="inline-flex justify-between items-center w-full rounded-md border border-gray-300 dark:border-gray-500 shadow-sm px-4 py-2 bg-white dark:bg-gray-800 text-sm text-gray-700 dark:text-gray-200 hover:bg-gray-50 dark:hover:bg-gray-850 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-offset-gray-100 dark:focus:ring-offset-gray-900 focus:ring-indigo-500 text-left">
{FILTER_OPERATIONS_DISPLAY_NAMES[props.selectedType]}
<ChevronDownIcon className="-mr-2 ml-2 h-4 w-4 text-gray-500 dark:text-gray-400" aria-hidden="true" />
</Menu.Button>
</div>
@ -58,12 +58,13 @@ export default function FilterOperatorSelector(props) {
>
<Menu.Items
static
className="z-10 origin-top-left absolute left-0 mt-2 w-full rounded-md shadow-lg bg-white dark:bg-gray-800 ring-1 ring-black ring-opacity-5 focus:outline-none"
className="z-10 origin-top-left absolute left-0 mt-2 rounded-md shadow-lg bg-white dark:bg-gray-800 ring-1 ring-black ring-opacity-5 focus:outline-none"
>
<div className="py-1">
{renderTypeItem(FILTER_OPERATIONS.is, true)}
{renderTypeItem(FILTER_OPERATIONS.isNot, supportsIsNot(filterName))}
{renderTypeItem(FILTER_OPERATIONS.contains, isFreeChoiceFilter(filterName))}
{renderTypeItem(FILTER_OPERATIONS.does_not_contain, isFreeChoiceFilter(filterName))}
</div>
</Menu.Items>
</Transition>

9
assets/js/dashboard/filters.js
View File

@ -13,7 +13,8 @@ import {
formattedFilters,
EVENT_PROPS_PREFIX,
getPropertyKeyFromFilterKey,
getLabel
getLabel,
FILTER_OPERATIONS_DISPLAY_NAMES
} from "./util/filters"
const WRAPSTATE = { unwrapped: 0, waiting: 1, wrapped: 2 }
@ -41,10 +42,10 @@ function filterText(query, [operation, filterKey, clauses]) {
const formattedFilter = formattedFilters[filterKey]
if (formattedFilter) {
return <>{formattedFilter} {operation} {clauses.map((value) => <b key={value}>{getLabel(query.labels, filterKey, value)}</b>).reduce((prev, curr) => [prev, ' or ', curr])} </>
return <>{formattedFilter} {FILTER_OPERATIONS_DISPLAY_NAMES[operation]} {clauses.map((value) => <b key={value}>{getLabel(query.labels, filterKey, value)}</b>).reduce((prev, curr) => [prev, ' or ', curr])} </>
} else if (filterKey.startsWith(EVENT_PROPS_PREFIX)) {
const propKey = getPropertyKeyFromFilterKey(filterKey)
return <>Property <b>{propKey}</b> {operation} {clauses.map((label) => <b key={label}>{label}</b>).reduce((prev, curr) => [prev, ' or ', curr])} </>
return <>Property <b>{propKey}</b> {FILTER_OPERATIONS_DISPLAY_NAMES[operation]} {clauses.map((label) => <b key={label}>{label}</b>).reduce((prev, curr) => [prev, ' or ', curr])} </>
}
throw new Error(`Unknown filter: ${filterKey}`)
@ -131,7 +132,7 @@ function Filters(props) {
window.addEventListener('resize', handleResize, false)
document.addEventListener('keyup', handleKeyup)
return () => {
window.removeEventListener('resize', handleResize, false)
document.removeEventListener("keyup", handleKeyup)

2
assets/js/dashboard/stats/modals/filter-modal-group.js
View File

@ -22,7 +22,7 @@ export default function FilterModalGroup({
[filterGroup, rows]
)
const showAddRow = site.flags.multiple_filters ? !['goal', 'hostname'].includes(filterGroup) : filterGroup == 'props'
const showAddRow = filterGroup == 'props'
const showTitle = filterGroup != 'props'
return (

2
assets/js/dashboard/stats/modals/filter-modal-props-row.js
View File

@ -33,7 +33,7 @@ export default function FilterModalPropsRow({
}
function fetchPropValueOptions(input) {
if (operation === FILTER_OPERATIONS.contains) {return Promise.resolve([])}
if ([FILTER_OPERATIONS.contains, FILTER_OPERATIONS.does_not_contain].includes(operation)) {return Promise.resolve([])}
return fetchSuggestions(apiPath(site, `/suggestions/prop_value`), query, input, [
FILTER_OPERATIONS.isNot, filterKey, ['(none)']
])

6
assets/js/dashboard/stats/modals/filter-modal-row.js
View File

@ -32,7 +32,7 @@ export default function FilterModalRow({
}
function fetchOptions(input) {
if (operation === FILTER_OPERATIONS.contains) {
if ([FILTER_OPERATIONS.contains, FILTER_OPERATIONS.does_not_contain].includes(operation)) {
return Promise.resolve([])
}
@ -43,14 +43,14 @@ export default function FilterModalRow({
return (
<div className="grid grid-cols-11 mt-1">
<div className="col-span-3 mr-2">
<div className="col-span-3">
<FilterOperatorSelector
forFilter={filterKey}
onSelect={(newOperation) => onUpdate([newOperation, filterKey, clauses], labels)}
selectedType={operation}
/>
</div>
<div className="col-span-8">
<div className="col-span-8 ml-2">
<Combobox
fetchOptions={fetchOptions}
freeChoice={isFreeChoiceFilter(filterKey)}

22
assets/js/dashboard/util/filters.js
View File

@ -23,22 +23,25 @@ export const NO_CONTAINS_OPERATOR = new Set(['goal', 'screen'].concat(FILTER_MOD
export const EVENT_PROPS_PREFIX = "props:"
export const FILTER_OPERATIONS = {
is: 'is',
isNot: 'is_not',
contains: 'contains',
is: 'is'
does_not_contain: 'does_not_contain'
};
export const OPERATION_PREFIX = {
export const FILTER_OPERATIONS_DISPLAY_NAMES = {
[FILTER_OPERATIONS.is]: 'is',
[FILTER_OPERATIONS.isNot]: 'is not',
[FILTER_OPERATIONS.contains]: 'contains',
[FILTER_OPERATIONS.does_not_contain]: 'does not contain'
}
const OPERATION_PREFIX = {
[FILTER_OPERATIONS.isNot]: '!',
[FILTER_OPERATIONS.contains]: '~',
[FILTER_OPERATIONS.is]: ''
};
export const BACKEND_OPERATION = {
[FILTER_OPERATIONS.is]: 'is',
[FILTER_OPERATIONS.isNot]: 'is_not',
[FILTER_OPERATIONS.contains]: 'matches'
}
export function supportsIsNot(filterName) {
return !['goal', 'prop_key'].includes(filterName)
@ -144,11 +147,8 @@ export function serializeApiFilters(filters) {
if (filterKey.startsWith(EVENT_PROPS_PREFIX) || EVENT_FILTER_KEYS.has(filterKey)) {
apiFilterKey = `event:${filterKey}`
}
if (operation == FILTER_OPERATIONS.contains) {
clauses = clauses.map((value) => value.includes('*') ? value : `**${value}**`)
}
clauses = clauses.map((value) => value.toString())
return [BACKEND_OPERATION[operation], apiFilterKey, clauses]
return [operation, apiFilterKey, clauses]
})
return JSON.stringify(apiFilters)

9
lib/plausible/stats/filters/query_parser.ex
View File

@ -61,6 +61,8 @@ defmodule Plausible.Stats.Filters.QueryParser do
defp parse_operator(["is_not" | _rest]), do: {:ok, :is_not}
defp parse_operator(["matches" | _rest]), do: {:ok, :matches}
defp parse_operator(["does_not_match" | _rest]), do: {:ok, :does_not_match}
defp parse_operator(["contains" | _rest]), do: {:ok, :contains}
defp parse_operator(["does_not_contain" | _rest]), do: {:ok, :does_not_contain}
defp parse_operator(filter), do: {:error, "Unknown operator for filter '#{inspect(filter)}'"}
defp parse_filter_key([_operator, filter_key | _rest] = filter) do
@ -69,10 +71,9 @@ defmodule Plausible.Stats.Filters.QueryParser do
defp parse_filter_key(filter), do: {:error, "Invalid filter '#{inspect(filter)}'"}
defp parse_filter_rest(:is, filter), do: parse_clauses_list(filter)
defp parse_filter_rest(:is_not, filter), do: parse_clauses_list(filter)
defp parse_filter_rest(:matches, filter), do: parse_clauses_list(filter)
defp parse_filter_rest(:does_not_match, filter), do: parse_clauses_list(filter)
defp parse_filter_rest(operator, filter)
when operator in [:is, :is_not, :matches, :does_not_match, :contains, :does_not_contain],
do: parse_clauses_list(filter)
defp parse_clauses_list([_operation, filter_key, list] = filter) when is_list(list) do
all_strings? = Enum.all?(list, &is_bitstring/1)

110
lib/plausible/stats/filters/where_builder.ex
View File

@ -72,35 +72,34 @@ defmodule Plausible.Stats.Filters.WhereBuilder do
dynamic([e], e.name in ^list)
end
defp add_filter(:events, _query, [:is, "event:goal", clauses]) do
{events, pages} = split_goals(clauses)
defp add_filter(:events, _query, [operation, "event:goal", clauses])
when operation in [:is, :matches] do
{events, pages, wildcard?} = split_goals(clauses)
dynamic([e], (e.pathname in ^pages and e.name == "pageview") or e.name in ^events)
end
if wildcard? do
event_clause =
if Enum.any?(events) do
dynamic([x], fragment("multiMatchAny(?, ?)", x.name, ^events))
else
dynamic([x], false)
end
defp add_filter(:events, _query, [:matches, "event:goal", clauses]) do
{events, pages} = split_goals(clauses, &page_regex/1)
page_clause =
if Enum.any?(pages) do
dynamic(
[x],
fragment("multiMatchAny(?, ?)", x.pathname, ^pages) and x.name == "pageview"
)
else
dynamic([x], false)
end
event_clause =
if Enum.any?(events) do
dynamic([x], fragment("multiMatchAny(?, ?)", x.name, ^events))
else
dynamic([x], false)
end
where_clause = dynamic([], ^event_clause or ^page_clause)
page_clause =
if Enum.any?(pages) do
dynamic(
[x],
fragment("multiMatchAny(?, ?)", x.pathname, ^pages) and x.name == "pageview"
)
else
dynamic([x], false)
end
where_clause = dynamic([], ^event_clause or ^page_clause)
dynamic([e], ^where_clause)
dynamic([e], ^where_clause)
else
dynamic([e], (e.pathname in ^pages and e.name == "pageview") or e.name in ^events)
end
end
defp add_filter(:events, _query, [_, "event:page" | _rest] = filter) do
@ -189,6 +188,44 @@ defmodule Plausible.Stats.Filters.WhereBuilder do
)
end
defp filter_custom_prop(prop_name, column_name, [:does_not_match, _, clauses]) do
regexes = Enum.map(clauses, &page_regex/1)
dynamic(
[t],
has_key(t, column_name, ^prop_name) and
fragment(
"not(arrayExists(k -> match(?, k), ?))",
get_by_key(t, column_name, ^prop_name),
^regexes
)
)
end
defp filter_custom_prop(prop_name, column_name, [:contains, _, clauses]) do
dynamic(
[t],
has_key(t, column_name, ^prop_name) and
fragment(
"multiSearchAny(?, ?)",
get_by_key(t, column_name, ^prop_name),
^clauses
)
)
end
defp filter_custom_prop(prop_name, column_name, [:does_not_contain, _, clauses]) do
dynamic(
[t],
has_key(t, column_name, ^prop_name) and
fragment(
"not(multiSearchAny(?, ?))",
get_by_key(t, column_name, ^prop_name),
^clauses
)
)
end
defp filter_field(db_field, [:matches, _key, glob_exprs]) do
page_regexes = Enum.map(glob_exprs, &page_regex/1)
dynamic([x], fragment("multiMatchAny(?, ?)", field(x, ^db_field), ^page_regexes))
@ -199,6 +236,14 @@ defmodule Plausible.Stats.Filters.WhereBuilder do
dynamic([x], fragment("not(multiMatchAny(?, ?))", field(x, ^db_field), ^page_regexes))
end
defp filter_field(db_field, [:contains, _key, values]) do
dynamic([x], fragment("multiSearchAny(?, ?)", field(x, ^db_field), ^values))
end
defp filter_field(db_field, [:does_not_contain, _key, values]) do
dynamic([x], fragment("not(multiSearchAny(?, ?))", field(x, ^db_field), ^values))
end
defp filter_field(db_field, [:is, _key, list]) do
list = Enum.map(list, &db_field_val(db_field, &1))
dynamic([x], field(x, ^db_field) in ^list)
@ -222,13 +267,14 @@ defmodule Plausible.Stats.Filters.WhereBuilder do
defp db_field_val(_, @not_set), do: ""
defp db_field_val(_, val), do: val
defp split_goals(clauses, map_fn \\ &Function.identity/1) do
groups =
Enum.group_by(clauses, fn {goal_type, _v} -> goal_type end, fn {_k, val} -> map_fn.(val) end)
defp split_goals(clauses) do
wildcard? = Enum.any?(clauses, fn {_, value} -> String.contains?(value, "*") end)
map_fn = if(wildcard?, do: &page_regex/1, else: &Function.identity/1)
{
Map.get(groups, :event, []),
Map.get(groups, :page, [])
}
clauses
|> Enum.reduce({[], [], wildcard?}, fn
{:event, value}, {event, page, wildcard?} -> {event ++ [map_fn.(value)], page, wildcard?}
{:page, value}, {event, page, wildcard?} -> {event, page ++ [map_fn.(value)], wildcard?}
end)
end
end

7
lib/plausible_web/controllers/stats_controller.ex
View File

@ -367,12 +367,7 @@ defmodule PlausibleWeb.StatsController do
defp shared_link_cookie_name(slug), do: "shared-link-" <> slug
defp get_flags(user, site),
do: %{
multiple_filters:
FunWithFlags.enabled?(:multiple_filters, for: user) ||
FunWithFlags.enabled?(:multiple_filters, for: site)
}
defp get_flags(_user, _site), do: %{}
defp is_dbip() do
on_ee do

241
test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
View File

@ -1746,4 +1746,245 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
assert json_response(conn, 200)["results"] == %{"conversion_rate" => %{"value" => 0}}
end
end
describe "with json filters" do
test "filtering by exact string", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, pathname: "/en*"),
build(:pageview, pathname: "/en*/page1"),
build(:pageview, pathname: "/en*/page2"),
build(:pageview, pathname: "/ena/page2"),
build(:pageview, pathname: "/pll/page1")
])
conn =
get(conn, "/api/v1/stats/aggregate", %{
"site_id" => site.domain,
"metrics" => "visitors",
"filters" => [
["is", "event:page", ["/en*"]]
]
})
assert json_response(conn, 200)["results"] == %{"visitors" => %{"value" => 1}}
end
test "filtering by goal", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, pathname: "/blog/post-1"),
build(:pageview, pathname: "/blog/post-2", user_id: @user_id),
build(:pageview, pathname: "/blog", user_id: @user_id),
build(:pageview, pathname: "/")
])
insert(:goal, %{site: site, page_path: "/blog"})
conn =
get(conn, "/api/v1/stats/aggregate", %{
"site_id" => site.domain,
"period" => "day",
"metrics" => "visitors,pageviews",
"filters" => [["is", "event:goal", ["Visit /blog"]]]
})
assert json_response(conn, 200)["results"] == %{
"visitors" => %{"value" => 1},
"pageviews" => %{"value" => 1}
}
end
test "filtering by wildcard goal", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, pathname: "/blog/post-1"),
build(:pageview, pathname: "/blog/post-2", user_id: @user_id),
build(:pageview, pathname: "/blog", user_id: @user_id),
build(:pageview, pathname: "/")
])
insert(:goal, %{site: site, page_path: "/blog**"})
conn =
get(conn, "/api/v1/stats/aggregate", %{
"site_id" => site.domain,
"period" => "day",
"metrics" => "visitors,pageviews",
"filters" => [["is", "event:goal", ["Visit /blog**"]]]
})
assert json_response(conn, 200)["results"] == %{
"visitors" => %{"value" => 2},
"pageviews" => %{"value" => 3}
}
end
test "contains", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, pathname: "/en*"),
build(:pageview, pathname: "/en*/page1"),
build(:pageview, pathname: "/en*/page2"),
build(:pageview, pathname: "/ena/page2"),
build(:pageview, pathname: "/pll/page1")
])
conn =
get(conn, "/api/v1/stats/aggregate", %{
"site_id" => site.domain,
"metrics" => "visitors",
"filters" => [
["contains", "event:page", ["/en*"]]
]
})
assert json_response(conn, 200)["results"] == %{"visitors" => %{"value" => 3}}
end
test "does not contain", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, pathname: "/en*"),
build(:pageview, pathname: "/en*/page1"),
build(:pageview, pathname: "/en*/page2"),
build(:pageview, pathname: "/ena/page2"),
build(:pageview, pathname: "/pll/page1")
])
conn =
get(conn, "/api/v1/stats/aggregate", %{
"site_id" => site.domain,
"metrics" => "visitors",
"filters" => [
["does_not_contain", "event:page", ["/en*"]]
]
})
assert json_response(conn, 200)["results"] == %{"visitors" => %{"value" => 2}}
end
test "matches custom event property", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
"meta.key": ["tier"],
"meta.value": ["large-1"]
),
build(:pageview,
"meta.key": ["tier"],
"meta.value": ["small-1"]
),
build(:pageview,
"meta.key": ["tier"],
"meta.value": ["small-1"]
),
build(:pageview,
"meta.key": ["tier"],
"meta.value": ["small-2"]
)
])
conn =
get(conn, "/api/v1/stats/aggregate", %{
"site_id" => site.domain,
"metrics" => "visitors",
"filters" => [
["matches", "event:props:tier", ["small*"]]
]
})
assert json_response(conn, 200)["results"] == %{"visitors" => %{"value" => 3}}
end
test "does_not_match custom event property", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
"meta.key": ["tier"],
"meta.value": ["large-1"]
),
build(:pageview,
"meta.key": ["tier"],
"meta.value": ["small-1"]
),
build(:pageview,
"meta.key": ["tier"],
"meta.value": ["small-1"]
),
build(:pageview,
"meta.key": ["tier"],
"meta.value": ["small-2"]
)
])
conn =
get(conn, "/api/v1/stats/aggregate", %{
"site_id" => site.domain,
"metrics" => "visitors",
"filters" => [
["does_not_match", "event:props:tier", ["small*"]]
]
})
assert json_response(conn, 200)["results"] == %{"visitors" => %{"value" => 1}}
end
test "contains custom event property", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
"meta.key": ["tier"],
"meta.value": ["large-1"]
),
build(:pageview,
"meta.key": ["tier"],
"meta.value": ["small-1"]
),
build(:pageview,
"meta.key": ["tier"],
"meta.value": ["small-1"]
),
build(:pageview,
"meta.key": ["tier"],
"meta.value": ["small-2"]
)
])
conn =
get(conn, "/api/v1/stats/aggregate", %{
"site_id" => site.domain,
"metrics" => "visitors",
"filters" => [
["contains", "event:props:tier", ["small"]]
]
})
assert json_response(conn, 200)["results"] == %{"visitors" => %{"value" => 3}}
end
test "does_not_contain custom event property", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
"meta.key": ["tier"],
"meta.value": ["large-1"]
),
build(:pageview,
"meta.key": ["tier"],
"meta.value": ["small-1"]
),
build(:pageview,
"meta.key": ["tier"],
"meta.value": ["small-1"]
),
build(:pageview,
"meta.key": ["tier"],
"meta.value": ["small-2"]
)
])
conn =
get(conn, "/api/v1/stats/aggregate", %{
"site_id" => site.domain,
"metrics" => "visitors",
"filters" => [
["does_not_contain", "event:props:tier", ["small"]]
]
})
assert json_response(conn, 200)["results"] == %{"visitors" => %{"value" => 1}}
end
end
end