plausible/analytics
1
1
Fork 0
mirror of https://github.com/plausible/analytics.git synced 2024-08-17 05:40:24 +03:00
Code Issues Packages Projects Releases Wiki Activity

APIv2: Cleanup (#4308)

Browse Source 
* Remove a dead method

* Move select_event/session_metrics to within QueryBuilder

* Make a method private

* Move page_regex util around

* Move utc_boundaries helper around

* Fixups for utc_boundaries

* Add legacy notices

* Move Stats.query method around

* include_sentry_replay_info consistently

* Move filters out of select_group_fields

* Collapse conditions under select_group_fields

* Shorten some imported methods

* Use dimension over dim in group_by

* Separate SQL query building from imported.ex

* props.ex -> legacy_dimensions.ex

* Move some query building out of Query.ex

* Remove unneeded method

* put_filter -> add_filter

* Remove some query setters

* Moduledoc

* Split out validations and import tests from query_test

* Move tests around

* Split event:goal tests from query_test

* Remove redundant filters

* Remove dead code

* Split special metrics tests from query_test

* Legacy module
This commit is contained in:
Karl-Aksel Puulmann 2024-07-09 14:31:45 +03:00 committed by GitHub
parent a181f3eab3
commit a9676546dc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
26 changed files with 2094 additions and 2112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
extra/lib/plausible/stats/funnel.ex
View File

@ -110,7 +110,7 @@ defmodule Plausible.Stats.Funnel do
%Plausible.Goal{page_path: pathname} when is_binary(pathname) ->
if String.contains?(pathname, "*") do
regex = Plausible.Stats.Base.page_regex(pathname)
regex = Plausible.Stats.Filters.Utils.page_regex(pathname)
dynamic([], fragment("match(pathname, ?)", ^regex))
else
dynamic([], fragment("pathname = ?", ^pathname))

2
lib/plausible/google/search_console/filters.ex
View File

@ -1,6 +1,6 @@
defmodule Plausible.Google.SearchConsole.Filters do
@moduledoc false
import Plausible.Stats.Base, only: [page_regex: 1]
import Plausible.Stats.Filters.Utils, only: [page_regex: 1]
def transform(property, plausible_filters) do
search_console_filters =

20
lib/plausible/stats.ex
View File

@ -15,6 +15,17 @@ defmodule Plausible.Stats do
use Plausible.DebugReplayInfo
def query(site, query) do
include_sentry_replay_info()
optimized_query = QueryOptimizer.optimize(query)
optimized_query
|> SQL.QueryBuilder.build(site)
|> ClickhouseRepo.all()
|> QueryResult.from(optimized_query)
end
def breakdown(site, query, metrics, pagination) do
include_sentry_replay_info()
Breakdown.breakdown(site, query, metrics, pagination)
@ -35,15 +46,6 @@ defmodule Plausible.Stats do
CurrentVisitors.current_visitors(site)
end
def query(site, query) do
optimized_query = QueryOptimizer.optimize(query)
optimized_query
|> SQL.QueryBuilder.build(site)
|> ClickhouseRepo.all()
|> QueryResult.from(optimized_query)
end
on_ee do
def funnel(site, query, funnel) do
include_sentry_replay_info()

6
lib/plausible/stats/aggregate.ex
View File

@ -1,4 +1,10 @@
defmodule Plausible.Stats.Aggregate do
@moduledoc """
Builds aggregate results for v1 of our stats API and dashboards.
Avoid adding new logic here - update QueryBuilder etc instead.
"""
use Plausible.ClickhouseRepo
use Plausible
import Plausible.Stats.Base

98
lib/plausible/stats/base.ex
View File

@ -1,10 +1,7 @@
defmodule Plausible.Stats.Base do
use Plausible.ClickhouseRepo
use Plausible
use Plausible.Stats.SQL.Fragments
alias Plausible.Stats.{Query, TableDecider, SQL}
alias Plausible.Timezones
alias Plausible.Stats.{TableDecider, SQL}
import Ecto.Query
def base_event_query(site, query) do
@ -29,7 +26,7 @@ defmodule Plausible.Stats.Base do
end
end
def query_events(site, query) do
defp query_events(site, query) do
q = from(e in "events_v2", where: ^SQL.WhereBuilder.build(:events, site, query))
on_ee do
@ -48,95 +45,4 @@ defmodule Plausible.Stats.Base do
q
end
def select_event_metrics(metrics) do
metrics
|> Enum.map(&SQL.Expression.event_metric/1)
|> Enum.reduce(%{}, &Map.merge/2)
end
def select_session_metrics(metrics, query) do
metrics
|> Enum.map(&SQL.Expression.session_metric(&1, query))
|> Enum.reduce(%{}, &Map.merge/2)
end
def filter_converted_sessions(db_query, site, query) do
if Query.has_event_filters?(query) do
converted_sessions =
from(e in query_events(site, query),
select: %{
session_id: fragment("DISTINCT ?", e.session_id),
_sample_factor: fragment("_sample_factor")
}
)
from(s in db_query,
join: cs in subquery(converted_sessions),
on: s.session_id == cs.session_id
)
else
db_query
end
end
defp beginning_of_time(candidate, native_stats_start_at) do
if Timex.after?(native_stats_start_at, candidate) do
native_stats_start_at
else
candidate
end
end
def utc_boundaries(%Query{period: "realtime", now: now}, site) do
last_datetime =
now
|> Timex.shift(seconds: 5)
|> beginning_of_time(site.native_stats_start_at)
|> NaiveDateTime.truncate(:second)
first_datetime =
now |> Timex.shift(minutes: -5) |> NaiveDateTime.truncate(:second)
{first_datetime, last_datetime}
end
def utc_boundaries(%Query{period: "30m", now: now}, site) do
last_datetime =
now
|> Timex.shift(seconds: 5)
|> beginning_of_time(site.native_stats_start_at)
|> NaiveDateTime.truncate(:second)
first_datetime =
now |> Timex.shift(minutes: -30) |> NaiveDateTime.truncate(:second)
{first_datetime, last_datetime}
end
def utc_boundaries(%Query{date_range: date_range}, site) do
{:ok, first} = NaiveDateTime.new(date_range.first, ~T[00:00:00])
first_datetime =
first
|> Timezones.to_utc_datetime(site.timezone)
|> beginning_of_time(site.native_stats_start_at)
{:ok, last} = NaiveDateTime.new(date_range.last |> Timex.shift(days: 1), ~T[00:00:00])
last_datetime = Timezones.to_utc_datetime(last, site.timezone)
{first_datetime, last_datetime}
end
def page_regex(expr) do
escaped =
expr
|> Regex.escape()
|> String.replace("\\|", "|")
|> String.replace("\\*\\*", ".*")
|> String.replace("\\*", ".*")
"^#{escaped}$"
end
end

6
lib/plausible/stats/breakdown.ex
View File

@ -1,4 +1,10 @@
defmodule Plausible.Stats.Breakdown do
@moduledoc """
Builds breakdown results for v1 of our stats API and dashboards.
Avoid adding new logic here - update QueryBuilder etc instead.
"""
use Plausible.ClickhouseRepo
use Plausible
use Plausible.Stats.SQL.Fragments

6
lib/plausible/stats/email_report.ex
View File

@ -40,7 +40,7 @@ defmodule Plausible.Stats.EmailReport do
end
defp put_top_5_pages(stats, site, query) do
query = Query.set_dimensions(query, ["event:page"])
query = Query.set(query, dimensions: ["event:page"])
pages = Stats.breakdown(site, query, [:visitors], {5, 1})
Map.put(stats, :pages, pages)
end
@ -48,8 +48,8 @@ defmodule Plausible.Stats.EmailReport do
defp put_top_5_sources(stats, site, query) do
query =
query
|> Query.put_filter([:is_not, "visit:source", ["Direct / None"]])
|> Query.set_dimensions(["visit:source"])
|> Query.add_filter([:is_not, "visit:source", ["Direct / None"]])
|> Query.set(dimensions: ["visit:source"])
sources = Stats.breakdown(site, query, [:visitors], {5, 1})

13
lib/plausible/stats/filters/utils.ex
View File

@ -76,8 +76,19 @@ defmodule Plausible.Stats.Filters.Utils do
events = Enum.map(event_goals, fn {_, event} -> event end)
page_regexes =
Enum.map(pageview_goals, fn {_, path} -> Plausible.Stats.Base.page_regex(path) end)
Enum.map(pageview_goals, fn {_, path} -> page_regex(path) end)
{events, page_regexes}
end
def page_regex(expr) do
escaped =
expr
|> Regex.escape()
|> String.replace("\\|", "|")
|> String.replace("\\*\\*", ".*")
|> String.replace("\\*", ".*")
"^#{escaped}$"
end
end

481
lib/plausible/stats/imported/imported.ex
View File

@ -3,17 +3,13 @@ defmodule Plausible.Stats.Imported do
use Plausible.Stats.SQL.Fragments
import Ecto.Query
import Plausible.Stats.Util, only: [shortname: 2]
import Plausible.Stats.Imported.SQL.Builder
alias Plausible.Stats.Filters
alias Plausible.Stats.Imported
alias Plausible.Stats.Query
alias Plausible.Stats.SQL.QueryBuilder
@no_ref "Direct / None"
@not_set "(not set)"
@none "(none)"
@property_to_table_mappings Imported.Base.property_to_table_mappings()
@goals_with_url Plausible.Imported.goals_with_url()
@ -302,479 +298,4 @@ defmodule Plausible.Stats.Imported do
|> Imported.Base.query_imported(query)
|> select_merge([i], %{total_visitors: fragment("sum(?)", i.visitors)})
end
defp select_imported_metrics(q, []), do: q
defp select_imported_metrics(q, [:visitors | rest]) do
q
|> select_merge([i], %{visitors: sum(i.visitors)})
|> select_imported_metrics(rest)
end
defp select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_custom_events", _}}} = q,
[:events | rest]
) do
q
|> select_merge([i], %{events: sum(i.events)})
|> select_imported_metrics(rest)
end
defp select_imported_metrics(q, [:events | rest]) do
q
|> select_merge([i], %{events: sum(i.pageviews)})
|> select_imported_metrics(rest)
end
defp select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,
[:visits | rest]
) do
q
|> select_merge([i], %{visits: sum(i.exits)})
|> select_imported_metrics(rest)
end
defp select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,
[:visits | rest]
) do
q
|> select_merge([i], %{visits: sum(i.entrances)})
|> select_imported_metrics(rest)
end
defp select_imported_metrics(q, [:visits | rest]) do
q
|> select_merge([i], %{visits: sum(i.visits)})
|> select_imported_metrics(rest)
end
defp select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_custom_events", _}}} = q,
[:pageviews | rest]
) do
q
|> select_merge([i], %{pageviews: 0})
|> select_imported_metrics(rest)
end
defp select_imported_metrics(q, [:pageviews | rest]) do
q
|> where([i], i.pageviews > 0)
|> select_merge([i], %{pageviews: sum(i.pageviews)})
|> select_imported_metrics(rest)
end
defp select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_pages", _}}} = q,
[:bounce_rate | rest]
) do
q
|> select_merge([i], %{
bounces: 0,
__internal_visits: 0
})
|> select_imported_metrics(rest)
end
defp select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,
[:bounce_rate | rest]
) do
q
|> select_merge([i], %{
bounces: sum(i.bounces),
__internal_visits: sum(i.entrances)
})
|> select_imported_metrics(rest)
end
defp select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,
[:bounce_rate | rest]
) do
q
|> select_merge([i], %{
bounces: sum(i.bounces),
__internal_visits: sum(i.exits)
})
|> select_imported_metrics(rest)
end
defp select_imported_metrics(q, [:bounce_rate | rest]) do
q
|> select_merge([i], %{
bounces: sum(i.bounces),
__internal_visits: sum(i.visits)
})
|> select_imported_metrics(rest)
end
defp select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_pages", _}}} = q,
[:visit_duration | rest]
) do
q
|> select_merge([i], %{
visit_duration: 0,
__internal_visits: 0
})
|> select_imported_metrics(rest)
end
defp select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,
[:visit_duration | rest]
) do
q
|> select_merge([i], %{
visit_duration: sum(i.visit_duration),
__internal_visits: sum(i.entrances)
})
|> select_imported_metrics(rest)
end
defp select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,
[:visit_duration | rest]
) do
q
|> select_merge([i], %{
visit_duration: sum(i.visit_duration),
__internal_visits: sum(i.exits)
})
|> select_imported_metrics(rest)
end
defp select_imported_metrics(q, [:visit_duration | rest]) do
q
|> select_merge([i], %{
visit_duration: sum(i.visit_duration),
__internal_visits: sum(i.visits)
})
|> select_imported_metrics(rest)
end
defp select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,
[:views_per_visit | rest]
) do
q
|> where([i], i.pageviews > 0)
|> select_merge([i], %{
pageviews: sum(i.pageviews),
__internal_visits: sum(i.entrances)
})
|> select_imported_metrics(rest)
end
defp select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,
[:views_per_visit | rest]
) do
q
|> where([i], i.pageviews > 0)
|> select_merge([i], %{
pageviews: sum(i.pageviews),
__internal_visits: sum(i.exits)
})
|> select_imported_metrics(rest)
end
defp select_imported_metrics(q, [:views_per_visit | rest]) do
q
|> where([i], i.pageviews > 0)
|> select_merge([i], %{
pageviews: sum(i.pageviews),
__internal_visits: sum(i.visits)
})
|> select_imported_metrics(rest)
end
defp select_imported_metrics(q, [_ | rest]) do
q
|> select_imported_metrics(rest)
end
defp group_imported_by(q, query) do
Enum.reduce(query.dimensions, q, fn dimension, q ->
dim = Plausible.Stats.Filters.without_prefix(dimension)
group_imported_by(q, dim, shortname(query, dimension), query)
end)
end
defp group_imported_by(q, dim, key, _query) when dim in [:source, :referrer] do
q
|> group_by([i], field(i, ^dim))
|> select_merge_as([i], %{
key => fragment("if(empty(?), ?, ?)", field(i, ^dim), @no_ref, field(i, ^dim))
})
end
defp group_imported_by(q, dim, key, _query)
when dim in [:utm_source, :utm_medium, :utm_campaign, :utm_term, :utm_content] do
q
|> group_by([i], field(i, ^dim))
|> where([i], fragment("not empty(?)", field(i, ^dim)))
|> select_merge_as([i], %{key => field(i, ^dim)})
end
defp group_imported_by(q, :page, key, _query) do
q
|> group_by([i], i.page)
|> select_merge_as([i], %{key => i.page, time_on_page: sum(i.time_on_page)})
end
defp group_imported_by(q, :country, key, _query) do
q
|> group_by([i], i.country)
|> where([i], i.country != "ZZ")
|> select_merge_as([i], %{key => i.country})
end
defp group_imported_by(q, :region, key, _query) do
q
|> group_by([i], i.region)
|> where([i], i.region != "")
|> select_merge_as([i], %{key => i.region})
end
defp group_imported_by(q, :city, key, _query) do
q
|> group_by([i], i.city)
|> where([i], i.city != 0 and not is_nil(i.city))
|> select_merge_as([i], %{key => i.city})
end
defp group_imported_by(q, dim, key, _query) when dim in [:device, :browser] do
q
|> group_by([i], field(i, ^dim))
|> select_merge_as([i], %{
key => fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim))
})
end
defp group_imported_by(q, :browser_version, key, _query) do
q
|> group_by([i], [i.browser_version])
|> select_merge_as([i], %{
key => fragment("if(empty(?), ?, ?)", i.browser_version, @not_set, i.browser_version)
})
end
defp group_imported_by(q, :os, key, _query) do
q
|> group_by([i], i.operating_system)
|> select_merge_as([i], %{
key => fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system)
})
end
defp group_imported_by(q, :os_version, key, _query) do
q
|> group_by([i], [i.operating_system_version])
|> select_merge_as([i], %{
key =>
fragment(
"if(empty(?), ?, ?)",
i.operating_system_version,
@not_set,
i.operating_system_version
)
})
end
defp group_imported_by(q, dim, key, _query) when dim in [:entry_page, :exit_page] do
q
|> group_by([i], field(i, ^dim))
|> select_merge_as([i], %{key => field(i, ^dim)})
end
defp group_imported_by(q, :name, key, _query) do
q
|> group_by([i], i.name)
|> select_merge_as([i], %{key => i.name})
end
defp group_imported_by(q, :url, key, _query) do
q
|> group_by([i], i.link_url)
|> select_merge_as([i], %{
key => fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)
})
end
defp group_imported_by(q, :path, key, _query) do
q
|> group_by([i], i.path)
|> select_merge_as([i], %{
key => fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)
})
end
defp group_imported_by(q, :month, key, _query) do
q
|> group_by([i], fragment("toStartOfMonth(?)", i.date))
|> select_merge_as([i], %{key => fragment("toStartOfMonth(?)", i.date)})
end
defp group_imported_by(q, :hour, key, _query) do
q
|> group_by([i], i.date)
|> select_merge_as([i], %{key => i.date})
end
defp group_imported_by(q, :week, key, query) do
q
|> group_by([i], weekstart_not_before(i.date, ^query.date_range.first))
|> select_merge_as([i], %{
key => weekstart_not_before(i.date, ^query.date_range.first)
})
end
defp group_imported_by(q, :day, key, _query) do
q
|> group_by([i], i.date)
|> select_merge_as([i], %{key => i.date})
end
defp select_joined_dimensions(q, query) do
Enum.reduce(query.dimensions, q, fn dimension, q ->
select_joined_dimension(q, dimension, shortname(query, dimension))
end)
end
defp select_joined_dimension(q, "visit:city", key) do
select_merge_as(q, [s, i], %{
key => fragment("greatest(?,?)", field(i, ^key), field(s, ^key))
})
end
defp select_joined_dimension(q, "time:" <> _, key) do
select_merge_as(q, [s, i], %{
key => fragment("greatest(?, ?)", field(i, ^key), field(s, ^key))
})
end
defp select_joined_dimension(q, _dimension, key) do
select_merge_as(q, [s, i], %{
key => fragment("if(empty(?), ?, ?)", field(s, ^key), field(i, ^key), field(s, ^key))
})
end
defp select_joined_metrics(q, []), do: q
# NOTE: Reverse-engineering the native data bounces and total visit
# durations to combine with imported data is inefficient. Instead both
# queries should fetch bounces/total_visit_duration and visits and be
# used as subqueries to a main query that then find the bounce rate/avg
# visit_duration.
defp select_joined_metrics(q, [:visits | rest]) do
q
|> select_merge_as([s, i], %{visits: s.visits + i.visits})
|> select_joined_metrics(rest)
end
defp select_joined_metrics(q, [:visitors | rest]) do
q
|> select_merge_as([s, i], %{visitors: s.visitors + i.visitors})
|> select_joined_metrics(rest)
end
defp select_joined_metrics(q, [:events | rest]) do
q
|> select_merge_as([s, i], %{events: s.events + i.events})
|> select_joined_metrics(rest)
end
defp select_joined_metrics(q, [:pageviews | rest]) do
q
|> select_merge_as([s, i], %{pageviews: s.pageviews + i.pageviews})
|> select_joined_metrics(rest)
end
defp select_joined_metrics(q, [:views_per_visit | rest]) do
q
|> select_merge_as([s, i], %{
views_per_visit:
fragment(
"if(? + ? > 0, round((? + ? * ?) / (? + ?), 2), 0)",
s.__internal_visits,
i.__internal_visits,
i.pageviews,
s.views_per_visit,
s.__internal_visits,
i.__internal_visits,
s.__internal_visits
)
})
|> select_joined_metrics(rest)
end
defp select_joined_metrics(q, [:bounce_rate | rest]) do
q
|> select_merge_as([s, i], %{
bounce_rate:
fragment(
"if(? + ? > 0, round(100 * (? + (? * ? / 100)) / (? + ?)), 0)",
s.__internal_visits,
i.__internal_visits,
i.bounces,
s.bounce_rate,
s.__internal_visits,
i.__internal_visits,
s.__internal_visits
)
})
|> select_joined_metrics(rest)
end
defp select_joined_metrics(q, [:visit_duration | rest]) do
q
|> select_merge_as([s, i], %{
visit_duration:
fragment(
"""
if(
? + ? > 0,
round((? + ? * ?) / (? + ?), 0),
0
)
""",
s.__internal_visits,
i.__internal_visits,
i.visit_duration,
s.visit_duration,
s.__internal_visits,
s.__internal_visits,
i.__internal_visits
)
})
|> select_joined_metrics(rest)
end
defp select_joined_metrics(q, [:sample_percent | rest]) do
q
|> select_merge_as([s, i], %{sample_percent: s.sample_percent})
|> select_joined_metrics(rest)
end
defp select_joined_metrics(q, [_ | rest]) do
q
|> select_joined_metrics(rest)
end
defp naive_dimension_join(q1, q2, metrics) do
from(a in subquery(q1),
full_join: b in subquery(q2),
on: a.dim0 == b.dim0,
select: %{}
)
|> select_merge_as([a, b], %{
dim0: fragment("if(? != 0, ?, ?)", a.dim0, a.dim0, b.dim0)
})
|> select_joined_metrics(metrics)
end
end

462
lib/plausible/stats/imported/sql/builder.ex Normal file
View File

@ -0,0 +1,462 @@
defmodule Plausible.Stats.Imported.SQL.Builder do
@moduledoc """
This module is responsible for generating SQL/Ecto expressions
for dimensions, filters and metrics used in import table queries
"""
use Plausible.Stats.SQL.Fragments
import Plausible.Stats.Util, only: [shortname: 2]
import Ecto.Query
@no_ref "Direct / None"
@not_set "(not set)"
@none "(none)"
def select_imported_metrics(q, []), do: q
def select_imported_metrics(q, [:visitors | rest]) do
q
|> select_merge([i], %{visitors: sum(i.visitors)})
|> select_imported_metrics(rest)
end
def select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_custom_events", _}}} = q,
[:events | rest]
) do
q
|> select_merge([i], %{events: sum(i.events)})
|> select_imported_metrics(rest)
end
def select_imported_metrics(q, [:events | rest]) do
q
|> select_merge([i], %{events: sum(i.pageviews)})
|> select_imported_metrics(rest)
end
def select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,
[:visits | rest]
) do
q
|> select_merge([i], %{visits: sum(i.exits)})
|> select_imported_metrics(rest)
end
def select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,
[:visits | rest]
) do
q
|> select_merge([i], %{visits: sum(i.entrances)})
|> select_imported_metrics(rest)
end
def select_imported_metrics(q, [:visits | rest]) do
q
|> select_merge([i], %{visits: sum(i.visits)})
|> select_imported_metrics(rest)
end
def select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_custom_events", _}}} = q,
[:pageviews | rest]
) do
q
|> select_merge([i], %{pageviews: 0})
|> select_imported_metrics(rest)
end
def select_imported_metrics(q, [:pageviews | rest]) do
q
|> where([i], i.pageviews > 0)
|> select_merge([i], %{pageviews: sum(i.pageviews)})
|> select_imported_metrics(rest)
end
def select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_pages", _}}} = q,
[:bounce_rate | rest]
) do
q
|> select_merge([i], %{
bounces: 0,
__internal_visits: 0
})
|> select_imported_metrics(rest)
end
def select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,
[:bounce_rate | rest]
) do
q
|> select_merge([i], %{
bounces: sum(i.bounces),
__internal_visits: sum(i.entrances)
})
|> select_imported_metrics(rest)
end
def select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,
[:bounce_rate | rest]
) do
q
|> select_merge([i], %{
bounces: sum(i.bounces),
__internal_visits: sum(i.exits)
})
|> select_imported_metrics(rest)
end
def select_imported_metrics(q, [:bounce_rate | rest]) do
q
|> select_merge([i], %{
bounces: sum(i.bounces),
__internal_visits: sum(i.visits)
})
|> select_imported_metrics(rest)
end
def select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_pages", _}}} = q,
[:visit_duration | rest]
) do
q
|> select_merge([i], %{
visit_duration: 0,
__internal_visits: 0
})
|> select_imported_metrics(rest)
end
def select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,
[:visit_duration | rest]
) do
q
|> select_merge([i], %{
visit_duration: sum(i.visit_duration),
__internal_visits: sum(i.entrances)
})
|> select_imported_metrics(rest)
end
def select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,
[:visit_duration | rest]
) do
q
|> select_merge([i], %{
visit_duration: sum(i.visit_duration),
__internal_visits: sum(i.exits)
})
|> select_imported_metrics(rest)
end
def select_imported_metrics(q, [:visit_duration | rest]) do
q
|> select_merge([i], %{
visit_duration: sum(i.visit_duration),
__internal_visits: sum(i.visits)
})
|> select_imported_metrics(rest)
end
def select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_entry_pages", _}}} = q,
[:views_per_visit | rest]
) do
q
|> where([i], i.pageviews > 0)
|> select_merge([i], %{
pageviews: sum(i.pageviews),
__internal_visits: sum(i.entrances)
})
|> select_imported_metrics(rest)
end
def select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,
[:views_per_visit | rest]
) do
q
|> where([i], i.pageviews > 0)
|> select_merge([i], %{
pageviews: sum(i.pageviews),
__internal_visits: sum(i.exits)
})
|> select_imported_metrics(rest)
end
def select_imported_metrics(q, [:views_per_visit | rest]) do
q
|> where([i], i.pageviews > 0)
|> select_merge([i], %{
pageviews: sum(i.pageviews),
__internal_visits: sum(i.visits)
})
|> select_imported_metrics(rest)
end
def select_imported_metrics(q, [_ | rest]) do
q
|> select_imported_metrics(rest)
end
def group_imported_by(q, query) do
Enum.reduce(query.dimensions, q, fn dimension, q ->
q
|> select_group_fields(dimension, shortname(query, dimension), query)
|> filter_group_values(dimension)
|> group_by([], selected_as(^shortname(query, dimension)))
end)
end
defp select_group_fields(q, dimension, key, _query)
when dimension in ["visit:source", "visit:referrer"] do
select_merge_as(q, [i], %{
key =>
fragment(
"if(empty(?), ?, ?)",
field(i, ^dim(dimension)),
@no_ref,
field(i, ^dim(dimension))
)
})
end
defp select_group_fields(q, "event:page", key, _query) do
select_merge_as(q, [i], %{key => i.page, time_on_page: sum(i.time_on_page)})
end
defp select_group_fields(q, dimension, key, _query)
when dimension in ["visit:device", "visit:browser"] do
select_merge_as(q, [i], %{
key =>
fragment(
"if(empty(?), ?, ?)",
field(i, ^dim(dimension)),
@not_set,
field(i, ^dim(dimension))
)
})
end
defp select_group_fields(q, "visit:browser_version", key, _query) do
select_merge_as(q, [i], %{
key => fragment("if(empty(?), ?, ?)", i.browser_version, @not_set, i.browser_version)
})
end
defp select_group_fields(q, "visit:os", key, _query) do
select_merge_as(q, [i], %{
key => fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system)
})
end
defp select_group_fields(q, "visit:os_version", key, _query) do
select_merge_as(q, [i], %{
key =>
fragment(
"if(empty(?), ?, ?)",
i.operating_system_version,
@not_set,
i.operating_system_version
)
})
end
defp select_group_fields(q, "event:props:url", key, _query) do
select_merge_as(q, [i], %{
key => fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)
})
end
defp select_group_fields(q, "event:props:path", key, _query) do
select_merge_as(q, [i], %{
key => fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)
})
end
defp select_group_fields(q, "time:month", key, _query) do
select_merge_as(q, [i], %{key => fragment("toStartOfMonth(?)", i.date)})
end
defp select_group_fields(q, dimension, key, _query)
when dimension in ["time:hour", "time:day"] do
select_merge_as(q, [i], %{key => i.date})
end
defp select_group_fields(q, "time:week", key, query) do
select_merge_as(q, [i], %{
key => weekstart_not_before(i.date, ^query.date_range.first)
})
end
defp select_group_fields(q, dimension, key, _query) do
select_merge_as(q, [i], %{key => field(i, ^dim(dimension))})
end
@utm_dimensions [
"visit:utm_source",
"visit:utm_medium",
"visit:utm_campaign",
"visit:utm_term",
"visit:utm_content"
]
defp filter_group_values(q, dimension) when dimension in @utm_dimensions do
dim = Plausible.Stats.Filters.without_prefix(dimension)
where(q, [i], fragment("not empty(?)", field(i, ^dim)))
end
defp filter_group_values(q, "visit:country"), do: where(q, [i], i.country != "ZZ")
defp filter_group_values(q, "visit:region"), do: where(q, [i], i.region != "")
defp filter_group_values(q, "visit:city"), do: where(q, [i], i.city != 0 and not is_nil(i.city))
defp filter_group_values(q, _dimension), do: q
def select_joined_dimensions(q, query) do
Enum.reduce(query.dimensions, q, fn dimension, q ->
select_joined_dimension(q, dimension, shortname(query, dimension))
end)
end
defp select_joined_dimension(q, "visit:city", key) do
select_merge_as(q, [s, i], %{
key => fragment("greatest(?,?)", field(i, ^key), field(s, ^key))
})
end
defp select_joined_dimension(q, "time:" <> _, key) do
select_merge_as(q, [s, i], %{
key => fragment("greatest(?, ?)", field(i, ^key), field(s, ^key))
})
end
defp select_joined_dimension(q, _dimension, key) do
select_merge_as(q, [s, i], %{
key => fragment("if(empty(?), ?, ?)", field(s, ^key), field(i, ^key), field(s, ^key))
})
end
def select_joined_metrics(q, []), do: q
# NOTE: Reverse-engineering the native data bounces and total visit
# durations to combine with imported data is inefficient. Instead both
# queries should fetch bounces/total_visit_duration and visits and be
# used as subqueries to a main query that then find the bounce rate/avg
# visit_duration.
def select_joined_metrics(q, [:visits | rest]) do
q
|> select_merge_as([s, i], %{visits: s.visits + i.visits})
|> select_joined_metrics(rest)
end
def select_joined_metrics(q, [:visitors | rest]) do
q
|> select_merge_as([s, i], %{visitors: s.visitors + i.visitors})
|> select_joined_metrics(rest)
end
def select_joined_metrics(q, [:events | rest]) do
q
|> select_merge_as([s, i], %{events: s.events + i.events})
|> select_joined_metrics(rest)
end
def select_joined_metrics(q, [:pageviews | rest]) do
q
|> select_merge_as([s, i], %{pageviews: s.pageviews + i.pageviews})
|> select_joined_metrics(rest)
end
def select_joined_metrics(q, [:views_per_visit | rest]) do
q
|> select_merge_as([s, i], %{
views_per_visit:
fragment(
"if(? + ? > 0, round((? + ? * ?) / (? + ?), 2), 0)",
s.__internal_visits,
i.__internal_visits,
i.pageviews,
s.views_per_visit,
s.__internal_visits,
i.__internal_visits,
s.__internal_visits
)
})
|> select_joined_metrics(rest)
end
def select_joined_metrics(q, [:bounce_rate | rest]) do
q
|> select_merge_as([s, i], %{
bounce_rate:
fragment(
"if(? + ? > 0, round(100 * (? + (? * ? / 100)) / (? + ?)), 0)",
s.__internal_visits,
i.__internal_visits,
i.bounces,
s.bounce_rate,
s.__internal_visits,
i.__internal_visits,
s.__internal_visits
)
})
|> select_joined_metrics(rest)
end
def select_joined_metrics(q, [:visit_duration | rest]) do
q
|> select_merge_as([s, i], %{
visit_duration:
fragment(
"""
if(
? + ? > 0,
round((? + ? * ?) / (? + ?), 0),
0
)
""",
s.__internal_visits,
i.__internal_visits,
i.visit_duration,
s.visit_duration,
s.__internal_visits,
s.__internal_visits,
i.__internal_visits
)
})
|> select_joined_metrics(rest)
end
def select_joined_metrics(q, [:sample_percent | rest]) do
q
|> select_merge_as([s, i], %{sample_percent: s.sample_percent})
|> select_joined_metrics(rest)
end
def select_joined_metrics(q, [_ | rest]) do
q
|> select_joined_metrics(rest)
end
def naive_dimension_join(q1, q2, metrics) do
from(a in subquery(q1),
full_join: b in subquery(q2),
on: a.dim0 == b.dim0,
select: %{}
)
|> select_merge_as([a, b], %{
dim0: fragment("if(? != 0, ?, ?)", a.dim0, a.dim0, b.dim0)
})
|> select_joined_metrics(metrics)
end
defp dim(dimension), do: Plausible.Stats.Filters.without_prefix(dimension)
end

13
lib/plausible/stats/props.ex → lib/plausible/stats/legacy/legacy_dimensions.ex
View File

@ -1,4 +1,7 @@
defmodule Plausible.Stats.Props do
defmodule Plausible.Stats.Legacy.Dimensions do
@moduledoc """
Deprecated module. See QueryParser for list of valid dimensions
"""
@event_props ["event:page", "event:name", "event:goal", "event:hostname"]
@session_props [
"visit:source",
@ -20,8 +23,8 @@ defmodule Plausible.Stats.Props do
"visit:browser_version"
]
def valid_prop?(prop) when prop in @event_props, do: true
def valid_prop?(prop) when prop in @session_props, do: true
def valid_prop?("event:props:" <> prop) when byte_size(prop) > 0, do: true
def valid_prop?(_), do: false
def valid?(prop) when prop in @event_props, do: true
def valid?(prop) when prop in @session_props, do: true
def valid?("event:props:" <> prop) when byte_size(prop) > 0, do: true
def valid?(_), do: false
end

180
lib/plausible/stats/legacy/legacy_query_builder.ex Normal file
View File

@ -0,0 +1,180 @@
defmodule Plausible.Stats.Legacy.QueryBuilder do
@moduledoc false
use Plausible
alias Plausible.Stats.{Filters, Interval, Query}
def from(site, params) do
now = NaiveDateTime.utc_now(:second)
query =
Query
|> struct!(now: now, timezone: site.timezone)
|> put_period(site, params)
|> put_dimensions(params)
|> put_interval(params)
|> put_parsed_filters(params)
|> Query.put_experimental_reduced_joins(site, params)
|> Query.put_imported_opts(site, params)
on_ee do
query = Plausible.Stats.Sampling.put_threshold(query, params)
end
query
end
defp put_period(query, site, %{"period" => "realtime"}) do
date = today(site.timezone)
struct!(query, period: "realtime", date_range: Date.range(date, date))
end
defp put_period(query, site, %{"period" => "day"} = params) do
date = parse_single_date(site.timezone, params)
struct!(query, period: "day", date_range: Date.range(date, date))
end
defp put_period(query, site, %{"period" => "7d"} = params) do
end_date = parse_single_date(site.timezone, params)
start_date = end_date |> Timex.shift(days: -6)
struct!(
query,
period: "7d",
date_range: Date.range(start_date, end_date)
)
end
defp put_period(query, site, %{"period" => "30d"} = params) do
end_date = parse_single_date(site.timezone, params)
start_date = end_date |> Timex.shift(days: -30)
struct!(query, period: "30d", date_range: Date.range(start_date, end_date))
end
defp put_period(query, site, %{"period" => "month"} = params) do
date = parse_single_date(site.timezone, params)
start_date = Timex.beginning_of_month(date)
end_date = Timex.end_of_month(date)
struct!(query,
period: "month",
date_range: Date.range(start_date, end_date)
)
end
defp put_period(query, site, %{"period" => "6mo"} = params) do
end_date =
parse_single_date(site.timezone, params)
|> Timex.end_of_month()
start_date =
Timex.shift(end_date, months: -5)
|> Timex.beginning_of_month()
struct!(query,
period: "6mo",
date_range: Date.range(start_date, end_date)
)
end
defp put_period(query, site, %{"period" => "12mo"} = params) do
end_date =
parse_single_date(site.timezone, params)
|> Timex.end_of_month()
start_date =
Timex.shift(end_date, months: -11)
|> Timex.beginning_of_month()
struct!(query,
period: "12mo",
date_range: Date.range(start_date, end_date)
)
end
defp put_period(query, site, %{"period" => "year"} = params) do
end_date =
parse_single_date(site.timezone, params)
|> Timex.end_of_year()
start_date = Timex.beginning_of_year(end_date)
struct!(query,
period: "year",
date_range: Date.range(start_date, end_date)
)
end
defp put_period(query, site, %{"period" => "all"}) do
now = today(site.timezone)
start_date = Plausible.Sites.stats_start_date(site) || now
struct!(query,
period: "all",
date_range: Date.range(start_date, now)
)
end
defp put_period(query, site, %{"period" => "custom", "from" => from, "to" => to} = params) do
new_params =
params
|> Map.drop(["from", "to"])
|> Map.put("date", Enum.join([from, to], ","))
put_period(query, site, new_params)
end
defp put_period(query, _site, %{"period" => "custom", "date" => date}) do
[from, to] = String.split(date, ",")
from_date = Date.from_iso8601!(String.trim(from))
to_date = Date.from_iso8601!(String.trim(to))
struct!(query,
period: "custom",
date_range: Date.range(from_date, to_date)
)
end
defp put_period(query, site, params) do
put_period(query, site, Map.merge(params, %{"period" => "30d"}))
end
defp put_dimensions(query, params) do
if not is_nil(params["property"]) do
struct!(query, dimensions: [params["property"]])
else
struct!(query, dimensions: Map.get(params, "dimensions", []))
end
end
defp put_interval(%{:period => "all"} = query, params) do
interval = Map.get(params, "interval", Interval.default_for_date_range(query.date_range))
struct!(query, interval: interval)
end
defp put_interval(query, params) do
interval = Map.get(params, "interval", Interval.default_for_period(query.period))
struct!(query, interval: interval)
end
defp put_parsed_filters(query, params) do
struct!(query, filters: Filters.parse(params["filters"]))
end
defp today(tz) do
Timex.now(tz) |> Timex.to_date()
end
defp parse_single_date(tz, params) do
case params["date"] do
"today" -> Timex.now(tz) |> Timex.to_date()
date when is_binary(date) -> Date.from_iso8601!(date)
_ -> today(tz)
end
end
end

218
lib/plausible/stats/query.ex
View File

@ -24,44 +24,31 @@ defmodule Plausible.Stats.Query do
}
require OpenTelemetry.Tracer, as: Tracer
alias Plausible.Stats.{Filters, Interval, Imported}
alias Plausible.Stats.{Filters, Imported, Legacy}
@type t :: %__MODULE__{}
@spec from(Plausible.Site.t(), map()) :: t()
def from(site, params) do
now = NaiveDateTime.utc_now(:second)
query =
__MODULE__
|> struct!(now: now, timezone: site.timezone)
|> put_experimental_reduced_joins(site, params)
|> put_period(site, params)
|> put_dimensions(params)
|> put_interval(params)
|> put_parsed_filters(params)
|> put_imported_opts(site, params)
on_ee do
query = Plausible.Stats.Sampling.put_threshold(query, params)
end
query
end
def build(site, params) do
with {:ok, query_data} <- Filters.QueryParser.parse(site, params) do
query =
struct!(__MODULE__, Map.to_list(query_data))
|> put_imported_opts(site, %{})
|> put_experimental_reduced_joins(site, params)
|> struct!(v2: true)
|> struct!(v2: true, now: NaiveDateTime.utc_now(:second))
{:ok, query}
end
end
defp put_experimental_reduced_joins(query, site, params) do
@doc """
Builds query from old-style params. New code should prefer Query.build
"""
@spec from(Plausible.Site.t(), map()) :: t()
def from(site, params) do
Legacy.QueryBuilder.from(site, params)
end
def put_experimental_reduced_joins(query, site, params) do
if Map.has_key?(params, "experimental_reduced_joins") do
struct!(query,
experimental_reduced_joins?: Map.get(params, "experimental_reduced_joins") == "true"
@ -73,147 +60,6 @@ defmodule Plausible.Stats.Query do
end
end
defp put_period(query, site, %{"period" => "realtime"}) do
date = today(site.timezone)
struct!(query, period: "realtime", date_range: Date.range(date, date))
end
defp put_period(query, site, %{"period" => "day"} = params) do
date = parse_single_date(site.timezone, params)
struct!(query, period: "day", date_range: Date.range(date, date))
end
defp put_period(query, site, %{"period" => "7d"} = params) do
end_date = parse_single_date(site.timezone, params)
start_date = end_date |> Timex.shift(days: -6)
struct!(
query,
period: "7d",
date_range: Date.range(start_date, end_date)
)
end
defp put_period(query, site, %{"period" => "30d"} = params) do
end_date = parse_single_date(site.timezone, params)
start_date = end_date |> Timex.shift(days: -30)
struct!(query, period: "30d", date_range: Date.range(start_date, end_date))
end
defp put_period(query, site, %{"period" => "month"} = params) do
date = parse_single_date(site.timezone, params)
start_date = Timex.beginning_of_month(date)
end_date = Timex.end_of_month(date)
struct!(query,
period: "month",
date_range: Date.range(start_date, end_date)
)
end
defp put_period(query, site, %{"period" => "6mo"} = params) do
end_date =
parse_single_date(site.timezone, params)
|> Timex.end_of_month()
start_date =
Timex.shift(end_date, months: -5)
|> Timex.beginning_of_month()
struct!(query,
period: "6mo",
date_range: Date.range(start_date, end_date)
)
end
defp put_period(query, site, %{"period" => "12mo"} = params) do
end_date =
parse_single_date(site.timezone, params)
|> Timex.end_of_month()
start_date =
Timex.shift(end_date, months: -11)
|> Timex.beginning_of_month()
struct!(query,
period: "12mo",
date_range: Date.range(start_date, end_date)
)
end
defp put_period(query, site, %{"period" => "year"} = params) do
end_date =
parse_single_date(site.timezone, params)
|> Timex.end_of_year()
start_date = Timex.beginning_of_year(end_date)
struct!(query,
period: "year",
date_range: Date.range(start_date, end_date)
)
end
defp put_period(query, site, %{"period" => "all"}) do
now = today(site.timezone)
start_date = Plausible.Sites.stats_start_date(site) || now
struct!(query,
period: "all",
date_range: Date.range(start_date, now)
)
end
defp put_period(query, site, %{"period" => "custom", "from" => from, "to" => to} = params) do
new_params =
params
|> Map.drop(["from", "to"])
|> Map.put("date", Enum.join([from, to], ","))
put_period(query, site, new_params)
end
defp put_period(query, _site, %{"period" => "custom", "date" => date}) do
[from, to] = String.split(date, ",")
from_date = Date.from_iso8601!(String.trim(from))
to_date = Date.from_iso8601!(String.trim(to))
struct!(query,
period: "custom",
date_range: Date.range(from_date, to_date)
)
end
defp put_period(query, site, params) do
put_period(query, site, Map.merge(params, %{"period" => "30d"}))
end
defp put_dimensions(query, params) do
if not is_nil(params["property"]) do
struct!(query, dimensions: [params["property"]])
else
struct!(query, dimensions: Map.get(params, "dimensions", []))
end
end
defp put_interval(%{:period => "all"} = query, params) do
interval = Map.get(params, "interval", Interval.default_for_date_range(query.date_range))
struct!(query, interval: interval)
end
defp put_interval(query, params) do
interval = Map.get(params, "interval", Interval.default_for_period(query.period))
struct!(query, interval: interval)
end
defp put_parsed_filters(query, params) do
struct!(query, filters: Filters.parse(params["filters"]))
end
def set(query, keywords) do
new_query = struct!(query, keywords)
@ -224,26 +70,7 @@ defmodule Plausible.Stats.Query do
end
end
@spec set_dimensions(t(), list(String.t())) :: t()
def set_dimensions(query, dimensions) do
query
|> struct!(dimensions: dimensions)
|> refresh_imported_opts()
end
def set_metrics(query, metrics) do
query
|> struct!(metrics: metrics)
|> refresh_imported_opts()
end
def set_order_by(query, order_by) do
query
|> struct!(order_by: order_by)
|> refresh_imported_opts()
end
def put_filter(query, filter) do
def add_filter(query, filter) do
query
|> struct!(filters: query.filters ++ [filter])
|> refresh_imported_opts()
@ -260,13 +87,6 @@ defmodule Plausible.Stats.Query do
|> refresh_imported_opts()
end
def exclude_imported(query) do
struct!(query,
include_imported: false,
skip_imported_reason: :manual_exclusion
)
end
defp refresh_imported_opts(query) do
put_imported_opts(query, nil, %{})
end
@ -289,19 +109,7 @@ defmodule Plausible.Stats.Query do
end)
end
defp today(tz) do
Timex.now(tz) |> Timex.to_date()
end
defp parse_single_date(tz, params) do
case params["date"] do
"today" -> Timex.now(tz) |> Timex.to_date()
date when is_binary(date) -> Date.from_iso8601!(date)
_ -> today(tz)
end
end
defp put_imported_opts(query, site, params) do
def put_imported_opts(query, site, params) do
requested? = params["with_imported"] == "true" || query.include.imports
latest_import_end_date =

16
lib/plausible/stats/sql/query_builder.ex
View File

@ -32,7 +32,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
from(
e in "events_v2",
where: ^SQL.WhereBuilder.build(:events, site, events_query),
select: ^Base.select_event_metrics(events_query.metrics)
select: ^select_event_metrics(events_query)
)
on_ee do
@ -73,7 +73,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
from(
e in "sessions_v2",
where: ^SQL.WhereBuilder.build(:sessions, site, sessions_query),
select: ^Base.select_session_metrics(sessions_query.metrics, sessions_query)
select: ^select_session_metrics(sessions_query)
)
on_ee do
@ -111,6 +111,18 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
end
end
defp select_event_metrics(query) do
query.metrics
|> Enum.map(&SQL.Expression.event_metric/1)
|> Enum.reduce(%{}, &Map.merge/2)
end
defp select_session_metrics(query) do
query.metrics
|> Enum.map(&SQL.Expression.session_metric(&1, query))
|> Enum.reduce(%{}, &Map.merge/2)
end
defp build_group_by(q, table, query) do
Enum.reduce(query.dimensions, q, &dimension_group_by(&2, table, query, &1))
end

3
lib/plausible/stats/sql/where_builder.ex
View File

@ -4,7 +4,8 @@ defmodule Plausible.Stats.SQL.WhereBuilder do
"""
import Ecto.Query
import Plausible.Stats.Base, only: [page_regex: 1, utc_boundaries: 2]
import Plausible.Stats.Time, only: [utc_boundaries: 2]
import Plausible.Stats.Filters.Utils, only: [page_regex: 1]
alias Plausible.Stats.Query

51
lib/plausible/stats/time.ex
View File

@ -4,6 +4,57 @@ defmodule Plausible.Stats.Time do
"""
alias Plausible.Stats.Query
alias Plausible.Timezones
def utc_boundaries(%Query{period: "realtime", now: now}, site) do
last_datetime =
now
|> Timex.shift(seconds: 5)
|> beginning_of_time(site.native_stats_start_at)
|> NaiveDateTime.truncate(:second)
first_datetime =
now |> Timex.shift(minutes: -5) |> NaiveDateTime.truncate(:second)
{first_datetime, last_datetime}
end
def utc_boundaries(%Query{period: "30m", now: now}, site) do
last_datetime =
now
|> Timex.shift(seconds: 5)
|> beginning_of_time(site.native_stats_start_at)
|> NaiveDateTime.truncate(:second)
first_datetime =
now |> Timex.shift(minutes: -30) |> NaiveDateTime.truncate(:second)
{first_datetime, last_datetime}
end
def utc_boundaries(%Query{date_range: date_range}, site) do
{:ok, first} = NaiveDateTime.new(date_range.first, ~T[00:00:00])
first_datetime =
first
|> Timezones.to_utc_datetime(site.timezone)
|> beginning_of_time(site.native_stats_start_at)
{:ok, last} = NaiveDateTime.new(date_range.last |> Timex.shift(days: 1), ~T[00:00:00])
last_datetime = Timezones.to_utc_datetime(last, site.timezone)
{first_datetime, last_datetime}
end
defp beginning_of_time(candidate, native_stats_start_at) do
if Timex.after?(native_stats_start_at, candidate) do
native_stats_start_at
else
candidate
end
end
def format_datetime(%Date{} = date), do: Date.to_string(date)
def format_datetime(%DateTime{} = datetime),

6
lib/plausible/stats/timeseries.ex
View File

@ -1,4 +1,10 @@
defmodule Plausible.Stats.Timeseries do
@moduledoc """
Builds timeseries results for v1 of our stats API and dashboards.
Avoid adding new logic here - update QueryBuilder etc instead.
"""
use Plausible
use Plausible.ClickhouseRepo
alias Plausible.Stats.{Query, QueryOptimizer, QueryResult, SQL}

4
lib/plausible_web/controllers/api/external_stats_controller.ex
View File

@ -76,7 +76,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
{:error,
"Property 'event:hostname' is currently not supported for breakdowns. Please provide a valid property for the breakdown endpoint: https://plausible.io/docs/stats-api#properties"}
Plausible.Stats.Props.valid_prop?(property) ->
Plausible.Stats.Legacy.Dimensions.valid?(property) ->
:ok
true ->
@ -363,7 +363,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
end
defp validate_filter(_site, [_, property | _]) do
if Plausible.Stats.Props.valid_prop?(property) do
if Plausible.Stats.Legacy.Dimensions.valid?(property) do
:ok
else
{:error,

8
lib/plausible_web/controllers/api/stats_controller.ex
View File

@ -777,7 +777,7 @@ defmodule PlausibleWeb.Api.StatsController do
query =
Query.from(site, params)
|> Query.put_filter(referrer_filter)
|> Query.add_filter(referrer_filter)
pagination = parse_pagination(params)
@ -907,9 +907,9 @@ defmodule PlausibleWeb.Api.StatsController do
total_pageviews_query =
query
|> Query.remove_filters(["visit:exit_page"])
|> Query.put_filter([:is, "event:page", pages])
|> Query.put_filter([:is, "event:name", ["pageview"]])
|> Query.set_dimensions(["event:page"])
|> Query.add_filter([:is, "event:page", pages])
|> Query.add_filter([:is, "event:name", ["pageview"]])
|> Query.set(dimensions: ["event:page"])
total_pageviews =
Stats.breakdown(site, total_pageviews_query, [:pageviews], {limit, 1})

2
test/plausible/stats/legacy_dashboard_filter_parser_test.exs
View File

@ -1,4 +1,4 @@
defmodule Plausible.Stats.LegacyDashboardFilterParserTest do
defmodule Plausible.Stats.Legacy.DashboardFilterParserTest do
use ExUnit.Case, async: true
alias Plausible.Stats.Filters.LegacyDashboardFilterParser

8
test/plausible/stats/query_test.exs
View File

@ -21,11 +21,11 @@ defmodule Plausible.Stats.QueryTest do
} do
q1 = %{now: %NaiveDateTime{}} = Query.from(site, %{"period" => "realtime"})
q2 = %{now: %NaiveDateTime{}} = Query.from(site, %{"period" => "30m"})
boundaries1 = Plausible.Stats.Base.utc_boundaries(q1, site)
boundaries2 = Plausible.Stats.Base.utc_boundaries(q2, site)
boundaries1 = Plausible.Stats.Time.utc_boundaries(q1, site)
boundaries2 = Plausible.Stats.Time.utc_boundaries(q2, site)
:timer.sleep(1500)
assert ^boundaries1 = Plausible.Stats.Base.utc_boundaries(q1, site)
assert ^boundaries2 = Plausible.Stats.Base.utc_boundaries(q2, site)
assert ^boundaries1 = Plausible.Stats.Time.utc_boundaries(q1, site)
assert ^boundaries2 = Plausible.Stats.Time.utc_boundaries(q2, site)
end
test "parses day format", %{site: site} do

133
test/plausible_web/controllers/api/external_stats_controller/query_goal_dimension_test.exs Normal file
View File

@ -0,0 +1,133 @@
defmodule PlausibleWeb.Api.ExternalStatsController.QueryGoalDimensionTest do
use PlausibleWeb.ConnCase
@user_id 1231
setup [:create_user, :create_new_site, :create_api_key, :use_api_key]
describe "breakdown by event:goal" do
test "returns custom event goals and pageview goals", %{conn: conn, site: site} do
insert(:goal, %{site: site, event_name: "Purchase"})
insert(:goal, %{site: site, page_path: "/test"})
populate_stats(site, [
build(:pageview,
timestamp: ~N[2021-01-01 00:00:01],
pathname: "/test"
),
build(:event,
name: "Purchase",
timestamp: ~N[2021-01-01 00:00:03]
),
build(:event,
name: "Purchase",
timestamp: ~N[2021-01-01 00:00:03]
)
])
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"date_range" => "all",
"metrics" => ["visitors"],
"dimensions" => ["event:goal"]
})
assert json_response(conn, 200)["results"] == [
%{"dimensions" => ["Purchase"], "metrics" => [2]},
%{"dimensions" => ["Visit /test"], "metrics" => [1]}
]
end
test "returns pageview goals containing wildcards", %{conn: conn, site: site} do
insert(:goal, %{site: site, page_path: "/**/post"})
insert(:goal, %{site: site, page_path: "/blog**"})
populate_stats(site, [
build(:pageview, pathname: "/blog", user_id: @user_id),
build(:pageview, pathname: "/blog/post-1", user_id: @user_id),
build(:pageview, pathname: "/blog/post-2", user_id: @user_id),
build(:pageview, pathname: "/blog/something/post"),
build(:pageview, pathname: "/different/page/post")
])
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"date_range" => "all",
"metrics" => ["visitors", "pageviews"],
"dimensions" => ["event:goal"],
"order_by" => [["pageviews", "desc"]]
})
assert json_response(conn, 200)["results"] == [
%{"dimensions" => ["Visit /blog**"], "metrics" => [2, 4]},
%{"dimensions" => ["Visit /**/post"], "metrics" => [2, 2]}
]
end
test "does not return goals that are not configured for the site", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, pathname: "/register"),
build(:event, name: "Signup")
])
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"date_range" => "all",
"metrics" => ["visitors", "pageviews"],
"dimensions" => ["event:goal"]
})
assert json_response(conn, 200)["results"] == []
end
test "returns conversion_rate in an event:goal breakdown", %{conn: conn, site: site} do
populate_stats(site, [
build(:event, name: "Signup", user_id: 1),
build(:event, name: "Signup", user_id: 1),
build(:pageview, pathname: "/blog"),
build(:pageview, pathname: "/blog/post"),
build(:pageview)
])
insert(:goal, %{site: site, event_name: "Signup"})
insert(:goal, %{site: site, page_path: "/blog**"})
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"date_range" => "all",
"metrics" => ["visitors", "events", "conversion_rate"],
"dimensions" => ["event:goal"]
})
assert json_response(conn, 200)["results"] == [
%{"dimensions" => ["Visit /blog**"], "metrics" => [2, 2, 50.0]},
%{"dimensions" => ["Signup"], "metrics" => [1, 2, 25.0]}
]
end
test "returns conversion_rate alone in an event:goal breakdown", %{conn: conn, site: site} do
populate_stats(site, [
build(:event, name: "Signup", user_id: 1),
build(:pageview)
])
insert(:goal, %{site: site, event_name: "Signup"})
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"date_range" => "all",
"metrics" => ["conversion_rate"],
"dimensions" => ["event:goal"]
})
assert json_response(conn, 200)["results"] == [
%{"dimensions" => ["Signup"], "metrics" => [50.0]}
]
end
end
end

695
test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs Normal file
View File

@ -0,0 +1,695 @@
defmodule PlausibleWeb.Api.ExternalStatsController.QueryImportedTest do
use PlausibleWeb.ConnCase
@user_id 1231
setup [:create_user, :create_new_site, :create_api_key, :use_api_key]
describe "aggregation with imported data" do
setup :create_site_import
test "does not count imported stats unless specified", %{
conn: conn,
site: site,
site_import: site_import
} do
populate_stats(site, site_import.id, [
build(:imported_visitors, date: ~D[2023-01-01]),
build(:pageview, timestamp: ~N[2023-01-01 00:00:00])
])
query_params = %{
"site_id" => site.domain,
"date_range" => "all",
"metrics" => ["pageviews"]
}
conn1 = post(conn, "/api/v2/query", query_params)
assert json_response(conn1, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}]
conn2 = post(conn, "/api/v2/query", Map.put(query_params, "include", %{"imports" => true}))
assert json_response(conn2, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}]
refute json_response(conn2, 200)["meta"]["warning"]
end
end
test "breaks down all metrics by visit:referrer with imported data", %{conn: conn, site: site} do
site_import =
insert(:site_import,
site: site,
start_date: ~D[2005-01-01],
end_date: Timex.today(),
source: :universal_analytics
)
populate_stats(site, site_import.id, [
build(:pageview, referrer: "site.com", timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, referrer: "site.com/1", timestamp: ~N[2021-01-01 00:00:00]),
build(:imported_sources,
referrer: "site.com",
date: ~D[2021-01-01],
visitors: 2,
visits: 2,
pageviews: 2,
bounces: 1,
visit_duration: 120
),
build(:imported_sources,
referrer: "site.com/2",
date: ~D[2021-01-01],
visitors: 2,
visits: 2,
pageviews: 2,
bounces: 2,
visit_duration: 0
),
build(:imported_sources,
date: ~D[2021-01-01],
visitors: 10,
visits: 11,
pageviews: 50,
bounces: 0,
visit_duration: 1100
)
])
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visitors", "visits", "pageviews", "bounce_rate", "visit_duration"],
"date_range" => "all",
"dimensions" => ["visit:referrer"],
"include" => %{"imports" => true}
})
assert json_response(conn, 200)["results"] == [
%{"dimensions" => ["Direct / None"], "metrics" => [10, 11, 50, 0.0, 100.0]},
%{"dimensions" => ["site.com"], "metrics" => [3, 3, 3, 67.0, 40.0]},
%{"dimensions" => ["site.com/2"], "metrics" => [2, 2, 2, 100.0, 0.0]},
%{"dimensions" => ["site.com/1"], "metrics" => [1, 1, 1, 100.0, 0.0]}
]
end
test "breaks down all metrics by visit:utm_source with imported data", %{conn: conn, site: site} do
site_import =
insert(:site_import,
site: site,
start_date: ~D[2005-01-01],
end_date: Timex.today(),
source: :universal_analytics
)
populate_stats(site, site_import.id, [
build(:pageview, utm_source: "SomeUTMSource", timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, utm_source: "SomeUTMSource-1", timestamp: ~N[2021-01-01 00:00:00]),
build(:imported_sources,
utm_source: "SomeUTMSource",
date: ~D[2021-01-01],
visitors: 2,
visits: 2,
pageviews: 2,
bounces: 1,
visit_duration: 120
),
build(:imported_sources,
utm_source: "SomeUTMSource-2",
date: ~D[2021-01-01],
visitors: 2,
visits: 2,
pageviews: 2,
bounces: 2,
visit_duration: 0
),
build(:imported_sources,
date: ~D[2021-01-01],
visitors: 10,
visits: 11,
pageviews: 50,
bounces: 0,
visit_duration: 1100
)
])
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visitors", "visits", "pageviews", "bounce_rate", "visit_duration"],
"date_range" => "all",
"dimensions" => ["visit:utm_source"],
"include" => %{"imports" => true}
})
%{"results" => results} = json_response(conn, 200)
assert results == [
%{"dimensions" => ["SomeUTMSource"], "metrics" => [3, 3, 3, 67.0, 40.0]},
%{"dimensions" => ["SomeUTMSource-2"], "metrics" => [2, 2, 2, 100.0, 0.0]},
%{"dimensions" => ["SomeUTMSource-1"], "metrics" => [1, 1, 1, 100.0, 0.0]}
]
end
test "pageviews breakdown by event:page - imported data having pageviews=0 and visitors=n should be bypassed",
%{conn: conn, site: site} do
site_import =
insert(:site_import,
site: site,
start_date: ~D[2005-01-01],
end_date: Timex.today(),
source: :universal_analytics
)
populate_stats(site, site_import.id, [
build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview,
pathname: "/plausible.io",
timestamp: ~N[2021-01-01 00:00:00]
),
build(:imported_pages,
page: "/skip-me",
date: ~D[2021-01-01],
visitors: 1,
pageviews: 0
),
build(:imported_pages,
page: "/include-me",
date: ~D[2021-01-01],
visitors: 1,
pageviews: 1
)
])
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["pageviews"],
"date_range" => "all",
"dimensions" => ["event:page"],
"include" => %{"imports" => true}
})
assert json_response(conn, 200)["results"] == [
%{"dimensions" => ["/"], "metrics" => [2]},
%{"dimensions" => ["/plausible.io"], "metrics" => [1]},
%{"dimensions" => ["/include-me"], "metrics" => [1]}
]
end
describe "breakdown by visit:exit_page with" do
setup %{site: site} do
site_import = insert(:site_import, site: site)
populate_stats(site, site_import.id, [
build(:pageview,
pathname: "/a",
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
pathname: "/a",
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview,
user_id: @user_id,
pathname: "/b",
timestamp: ~N[2021-01-01 00:35:00]
),
build(:imported_exit_pages,
exit_page: "/b",
exits: 3,
visitors: 2,
pageviews: 5,
date: ~D[2021-01-01]
)
])
end
test "can query with visit:exit_page dimension", %{conn: conn, site: site} do
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visits"],
"date_range" => "all",
"dimensions" => ["visit:exit_page"],
"include" => %{"imports" => true}
})
%{"results" => results} = json_response(conn, 200)
assert results == [
%{"dimensions" => ["/b"], "metrics" => [4]},
%{"dimensions" => ["/a"], "metrics" => [1]}
]
end
end
describe "imported data" do
test "returns screen sizes breakdown when filtering by screen size", %{conn: conn, site: site} do
site_import = insert(:site_import, site: site)
populate_stats(site, site_import.id, [
build(:pageview,
timestamp: ~N[2021-01-01 00:00:01],
screen_size: "Mobile"
),
build(:imported_devices,
device: "Mobile",
visitors: 3,
pageviews: 5,
date: ~D[2021-01-01]
)
])
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visitors", "pageviews"],
"date_range" => "all",
"dimensions" => ["visit:device"],
"filters" => [
["is", "visit:device", ["Mobile"]]
],
"include" => %{"imports" => true}
})
%{"results" => results} = json_response(conn, 200)
assert results == [%{"dimensions" => ["Mobile"], "metrics" => [4, 6]}]
end
test "returns custom event goals and pageview goals", %{conn: conn, site: site} do
insert(:goal, site: site, event_name: "Purchase")
insert(:goal, site: site, page_path: "/test")
site_import = insert(:site_import, site: site)
populate_stats(site, site_import.id, [
build(:pageview,
timestamp: ~N[2021-01-01 00:00:01],
pathname: "/test"
),
build(:event,
name: "Purchase",
timestamp: ~N[2021-01-01 00:00:03]
),
build(:event,
name: "Purchase",
timestamp: ~N[2021-01-01 00:00:03]
),
build(:imported_custom_events,
name: "Purchase",
visitors: 3,
events: 5,
date: ~D[2021-01-01]
),
build(:imported_pages,
page: "/test",
visitors: 2,
pageviews: 2,
date: ~D[2021-01-01]
),
build(:imported_visitors, visitors: 5, date: ~D[2021-01-01])
])
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"date_range" => "all",
"dimensions" => ["event:goal"],
"metrics" => ["visitors", "events", "pageviews", "conversion_rate"],
"include" => %{"imports" => true}
})
assert json_response(conn, 200)["results"] == [
%{"dimensions" => ["Purchase"], "metrics" => [5, 7, 0, 62.5]},
%{"dimensions" => ["Visit /test"], "metrics" => [3, 3, 3, 37.5]}
]
end
test "pageviews are returned as events for breakdown reports other than custom events", %{
conn: conn,
site: site
} do
site_import = insert(:site_import, site: site)
populate_stats(site, site_import.id, [
build(:imported_browsers, browser: "Chrome", pageviews: 1, date: ~D[2021-01-01]),
build(:imported_devices, device: "Desktop", pageviews: 1, date: ~D[2021-01-01]),
build(:imported_entry_pages, entry_page: "/test", pageviews: 1, date: ~D[2021-01-01]),
build(:imported_exit_pages, exit_page: "/test", pageviews: 1, date: ~D[2021-01-01]),
build(:imported_locations, country: "EE", pageviews: 1, date: ~D[2021-01-01]),
build(:imported_operating_systems,
operating_system: "Mac",
pageviews: 1,
date: ~D[2021-01-01]
),
build(:imported_pages, page: "/test", pageviews: 1, date: ~D[2021-01-01]),
build(:imported_sources, source: "Google", pageviews: 1, date: ~D[2021-01-01])
])
breakdown_and_first = fn dimension ->
conn
|> post("/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["events"],
"date_range" => ["2021-01-01", "2021-01-01"],
"dimensions" => [dimension],
"include" => %{"imports" => true}
})
|> json_response(200)
|> Map.get("results")
|> List.first()
end
assert %{"dimensions" => ["Chrome"], "metrics" => [1]} =
breakdown_and_first.("visit:browser")
assert %{"dimensions" => ["Desktop"], "metrics" => [1]} =
breakdown_and_first.("visit:device")
assert %{"dimensions" => ["EE"], "metrics" => [1]} = breakdown_and_first.("visit:country")
assert %{"dimensions" => ["Mac"], "metrics" => [1]} = breakdown_and_first.("visit:os")
assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("event:page")
assert %{"dimensions" => ["Google"], "metrics" => [1]} =
breakdown_and_first.("visit:source")
end
for goal_name <- Plausible.Imported.goals_with_url() do
test "returns url breakdown for #{goal_name} goal", %{conn: conn, site: site} do
insert(:goal, event_name: unquote(goal_name), site: site)
site_import = insert(:site_import, site: site)
populate_stats(site, site_import.id, [
build(:event,
name: unquote(goal_name),
"meta.key": ["url"],
"meta.value": ["https://one.com"]
),
build(:imported_custom_events,
name: unquote(goal_name),
visitors: 2,
events: 5,
link_url: "https://one.com"
),
build(:imported_custom_events,
name: unquote(goal_name),
visitors: 5,
events: 10,
link_url: "https://two.com"
),
build(:imported_custom_events,
name: "some goal",
visitors: 5,
events: 10
),
build(:imported_visitors, visitors: 9)
])
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visitors", "events", "conversion_rate"],
"date_range" => "all",
"dimensions" => ["event:props:url"],
"filters" => [
["is", "event:goal", [unquote(goal_name)]]
],
"include" => %{"imports" => true}
})
assert json_response(conn, 200)["results"] == [
%{"dimensions" => ["https://two.com"], "metrics" => [5, 10, 50]},
%{"dimensions" => ["https://one.com"], "metrics" => [3, 6, 30]}
]
refute json_response(conn, 200)["meta"]["warning"]
end
end
for goal_name <- Plausible.Imported.goals_with_path() do
test "returns path breakdown for #{goal_name} goal", %{conn: conn, site: site} do
insert(:goal, event_name: unquote(goal_name), site: site)
site_import = insert(:site_import, site: site)
populate_stats(site, site_import.id, [
build(:event,
name: unquote(goal_name),
"meta.key": ["path"],
"meta.value": ["/one"]
),
build(:imported_custom_events,
name: unquote(goal_name),
visitors: 2,
events: 5,
path: "/one"
),
build(:imported_custom_events,
name: unquote(goal_name),
visitors: 5,
events: 10,
path: "/two"
),
build(:imported_custom_events,
name: "some goal",
visitors: 5,
events: 10
),
build(:imported_visitors, visitors: 9)
])
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visitors", "events", "conversion_rate"],
"date_range" => "all",
"dimensions" => ["event:props:path"],
"filters" => [
["is", "event:goal", [unquote(goal_name)]]
],
"include" => %{"imports" => true}
})
assert json_response(conn, 200)["results"] == [
%{"dimensions" => ["/two"], "metrics" => [5, 10, 50]},
%{"dimensions" => ["/one"], "metrics" => [3, 6, 30]}
]
refute json_response(conn, 200)["meta"]["warning"]
end
end
test "adds a warning when query params are not supported for imported data", %{
conn: conn,
site: site
} do
site_import = insert(:site_import, site: site)
insert(:goal, event_name: "Signup", site: site)
populate_stats(site, site_import.id, [
build(:event,
name: "Signup",
"meta.key": ["package"],
"meta.value": ["large"]
),
build(:imported_visitors, visitors: 9)
])
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visitors"],
"date_range" => "all",
"dimensions" => ["event:props:package"],
"filters" => [
["is", "event:goal", ["Signup"]]
],
"include" => %{"imports" => true}
})
assert json_response(conn, 200)["results"] == [
%{"dimensions" => ["large"], "metrics" => [1]}
]
assert json_response(conn, 200)["meta"]["warning"] =~
"Imported stats are not included in the results because query parameters are not supported."
end
test "does not add a warning when there are no site imports", %{conn: conn, site: site} do
insert(:goal, event_name: "Signup", site: site)
populate_stats(site, [
build(:event,
name: "Signup",
"meta.key": ["package"],
"meta.value": ["large"]
)
])
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visitors"],
"date_range" => "all",
"dimensions" => ["event:props:package"],
"filters" => [
["is", "event:goal", ["Signup"]]
],
"include" => %{"imports" => true}
})
refute json_response(conn, 200)["meta"]["warning"]
end
test "does not add a warning when import is out of queried date range", %{
conn: conn,
site: site
} do
site_import = insert(:site_import, site: site, end_date: Date.add(Date.utc_today(), -3))
insert(:goal, event_name: "Signup", site: site)
populate_stats(site, site_import.id, [
build(:event,
name: "Signup",
"meta.key": ["package"],
"meta.value": ["large"]
),
build(:imported_visitors, visitors: 9)
])
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visitors"],
"date_range" => "day",
"dimensions" => ["event:props:package"],
"filters" => [
["is", "event:goal", ["Signup"]]
],
"include" => %{"imports" => true}
})
refute json_response(conn, 200)["meta"]["warning"]
end
test "applies multiple filters if the properties belong to the same table", %{
conn: conn,
site: site
} do
site_import = insert(:site_import, site: site)
populate_stats(site, site_import.id, [
build(:imported_sources, source: "Google", utm_medium: "organic", utm_term: "one"),
build(:imported_sources, source: "Twitter", utm_medium: "organic", utm_term: "two"),
build(:imported_sources,
source: "Facebook",
utm_medium: "something_else",
utm_term: "one"
)
])
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visitors"],
"date_range" => "day",
"dimensions" => ["visit:source"],
"filters" => [
["is", "visit:utm_medium", ["organic"]],
["is", "visit:utm_term", ["one"]]
],
"include" => %{"imports" => true}
})
assert json_response(conn, 200)["results"] == [
%{"dimensions" => ["Google"], "metrics" => [1]}
]
end
test "ignores imported data if filtered property belongs to a different table than the breakdown property",
%{
conn: conn,
site: site
} do
site_import = insert(:site_import, site: site)
populate_stats(site, site_import.id, [
build(:imported_sources, source: "Google"),
build(:imported_devices, device: "Desktop")
])
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visitors"],
"date_range" => "day",
"dimensions" => ["visit:source"],
"filters" => [
["is", "visit:device", ["Desktop"]]
],
"include" => %{"imports" => true}
})
assert %{
"results" => [],
"meta" => meta
} = json_response(conn, 200)
assert meta["warning"] =~ "Imported stats are not included in the results"
end
test "imported country, region and city data",
%{
conn: conn,
site: site
} do
site_import = insert(:site_import, site: site)
populate_stats(site, site_import.id, [
build(:pageview,
timestamp: ~N[2021-01-01 00:15:00],
country_code: "DE",
subdivision1_code: "DE-BE",
city_geoname_id: 2_950_159
),
build(:pageview,
timestamp: ~N[2021-01-01 00:15:00],
country_code: "DE",
subdivision1_code: "DE-BE",
city_geoname_id: 2_950_159
),
build(:pageview,
timestamp: ~N[2021-01-01 00:15:00],
country_code: "EE",
subdivision1_code: "EE-37",
city_geoname_id: 588_409
),
build(:imported_locations, country: "EE", region: "EE-37", city: 588_409, visitors: 33)
])
for {dimension, stats_value, imports_value} <- [
{"visit:country", "DE", "EE"},
{"visit:region", "DE-BE", "EE-37"},
{"visit:city", 2_950_159, 588_409}
] do
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visitors"],
"date_range" => "all",
"dimensions" => [dimension],
"include" => %{"imports" => true}
})
assert json_response(conn, 200)["results"] == [
%{"dimensions" => [imports_value], "metrics" => [34]},
%{"dimensions" => [stats_value], "metrics" => [2]}
]
end
end
end
end

193
test/plausible_web/controllers/api/external_stats_controller/query_special_metrics_test.exs Normal file
View File

@ -0,0 +1,193 @@
defmodule PlausibleWeb.Api.ExternalStatsController.QuerySpecialMetricsTest do
use PlausibleWeb.ConnCase
setup [:create_user, :create_new_site, :create_api_key, :use_api_key]
test "returns conversion_rate in a goal filtered custom prop breakdown", %{
conn: conn,
site: site
} do
populate_stats(site, [
build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]),
build(:pageview, pathname: "/blog/2", "meta.key": ["author"], "meta.value": ["Uku"]),
build(:pageview, pathname: "/blog/3", "meta.key": ["author"], "meta.value": ["Uku"]),
build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Marko"]),
build(:pageview,
pathname: "/blog/2",
"meta.key": ["author"],
"meta.value": ["Marko"],
user_id: 1
),
build(:pageview,
pathname: "/blog/3",
"meta.key": ["author"],
"meta.value": ["Marko"],
user_id: 1
),
build(:pageview, pathname: "/blog"),
build(:pageview, "meta.key": ["author"], "meta.value": ["Marko"]),
build(:pageview)
])
insert(:goal, %{site: site, page_path: "/blog**"})
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"date_range" => "all",
"filters" => [["matches", "event:goal", ["Visit /blog**"]]],
"metrics" => ["visitors", "events", "conversion_rate"],
"dimensions" => ["event:props:author"]
})
assert json_response(conn, 200)["results"] == [
%{"dimensions" => ["Uku"], "metrics" => [3, 3, 37.5]},
%{"dimensions" => ["Marko"], "metrics" => [2, 3, 25.0]},
%{"dimensions" => ["(none)"], "metrics" => [1, 1, 12.5]}
]
end
test "returns conversion_rate alone in a goal filtered custom prop breakdown", %{
conn: conn,
site: site
} do
populate_stats(site, [
build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]),
build(:pageview)
])
insert(:goal, %{site: site, page_path: "/blog**"})
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["conversion_rate"],
"date_range" => "all",
"dimensions" => ["event:props:author"],
"filters" => [["matches", "event:goal", ["Visit /blog**"]]]
})
%{"results" => results} = json_response(conn, 200)
assert results == [
%{"dimensions" => ["Uku"], "metrics" => [50]}
]
end
test "returns conversion_rate in a goal filtered event:page breakdown", %{
conn: conn,
site: site
} do
populate_stats(site, [
build(:event, pathname: "/en/register", name: "pageview"),
build(:event, pathname: "/en/register", name: "Signup"),
build(:event, pathname: "/en/register", name: "Signup"),
build(:event, pathname: "/it/register", name: "Signup", user_id: 1),
build(:event, pathname: "/it/register", name: "Signup", user_id: 1),
build(:event, pathname: "/it/register", name: "pageview")
])
insert(:goal, %{site: site, event_name: "Signup"})
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"date_range" => "all",
"dimensions" => ["event:page"],
"filters" => [["is", "event:goal", ["Signup"]]],
"metrics" => ["visitors", "events", "group_conversion_rate"]
})
assert json_response(conn, 200)["results"] == [
%{"dimensions" => ["/en/register"], "metrics" => [2, 2, 66.7]},
%{"dimensions" => ["/it/register"], "metrics" => [1, 2, 50.0]}
]
end
test "returns conversion_rate alone in a goal filtered event:page breakdown", %{
conn: conn,
site: site
} do
populate_stats(site, [
build(:event, pathname: "/en/register", name: "pageview"),
build(:event, pathname: "/en/register", name: "Signup")
])
insert(:goal, %{site: site, event_name: "Signup"})
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"date_range" => "all",
"filters" => [["is", "event:goal", ["Signup"]]],
"metrics" => ["group_conversion_rate"],
"dimensions" => ["event:page"]
})
assert json_response(conn, 200)["results"] == [
%{"dimensions" => ["/en/register"], "metrics" => [50.0]}
]
end
test "returns conversion_rate in a multi-goal filtered visit:screen_size breakdown", %{
conn: conn,
site: site
} do
populate_stats(site, [
build(:event, screen_size: "Mobile", name: "pageview"),
build(:event, screen_size: "Mobile", name: "AddToCart"),
build(:event, screen_size: "Mobile", name: "AddToCart"),
build(:event, screen_size: "Desktop", name: "AddToCart", user_id: 1),
build(:event, screen_size: "Desktop", name: "Purchase", user_id: 1),
build(:event, screen_size: "Desktop", name: "pageview")
])
# Make sure that revenue goals are treated the same
# way as regular custom event goals
insert(:goal, %{site: site, event_name: "Purchase", currency: :EUR})
insert(:goal, %{site: site, event_name: "AddToCart"})
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visitors", "events", "group_conversion_rate"],
"date_range" => "all",
"dimensions" => ["visit:device"],
"filters" => [["is", "event:goal", ["AddToCart", "Purchase"]]]
})
%{"results" => results} = json_response(conn, 200)
assert results == [
%{"dimensions" => ["Mobile"], "metrics" => [2, 2, 66.7]},
%{"dimensions" => ["Desktop"], "metrics" => [1, 2, 50]}
]
end
test "returns conversion_rate alone in a goal filtered visit:screen_size breakdown", %{
conn: conn,
site: site
} do
populate_stats(site, [
build(:event, screen_size: "Mobile", name: "pageview"),
build(:event, screen_size: "Mobile", name: "AddToCart")
])
insert(:goal, %{site: site, event_name: "AddToCart"})
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["conversion_rate"],
"date_range" => "all",
"dimensions" => ["visit:device"],
"filters" => [["is", "event:goal", ["AddToCart"]]]
})
%{"results" => results} = json_response(conn, 200)
assert results == [
%{"dimensions" => ["Mobile"], "metrics" => [50]}
]
end
end

1346
test/plausible_web/controllers/api/external_stats_controller/query_test.exs
View File

File diff suppressed because it is too large Load Diff

234
test/plausible_web/controllers/api/external_stats_controller/query_validations_test.exs Normal file
View File

@ -0,0 +1,234 @@
defmodule PlausibleWeb.Api.ExternalStatsController.QueryValidationsTest do
use PlausibleWeb.ConnCase
alias Plausible.Billing.Feature
setup [:create_user, :create_new_site, :create_api_key, :use_api_key]
describe "feature access" do
test "cannot break down by a custom prop without access to the props feature", %{
conn: conn,
user: user,
site: site
} do
ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)
insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visitors"],
"date_range" => "all",
"dimensions" => ["event:props:author"]
})
assert json_response(conn, 400)["error"] ==
"The owner of this site does not have access to the custom properties feature"
end
test "can break down by an internal prop key without access to the props feature", %{
conn: conn,
user: user,
site: site
} do
ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)
insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visitors"],
"date_range" => "all",
"dimensions" => ["event:props:path"]
})
assert json_response(conn, 200)["results"]
end
test "cannot filter by a custom prop without access to the props feature", %{
conn: conn,
user: user,
site: site
} do
ep =
insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)
insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visitors"],
"date_range" => "all",
"dimensions" => ["visit:source"],
"filters" => [["is", "event:props:author", ["Uku"]]]
})
assert json_response(conn, 400)["error"] ==
"The owner of this site does not have access to the custom properties feature"
end
test "can filter by an internal prop key without access to the props feature", %{
conn: conn,
user: user,
site: site
} do
ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)
insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visitors"],
"date_range" => "all",
"dimensions" => ["visit:source"],
"filters" => [["is", "event:props:url", ["whatever"]]]
})
assert json_response(conn, 200)["results"]
end
end
describe "param validation" do
test "does not allow querying conversion_rate without a goal filter", %{
conn: conn,
site: site
} do
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["conversion_rate"],
"date_range" => "all",
"dimensions" => ["event:page"],
"filters" => [["is", "event:props:author", ["Uku"]]]
})
assert json_response(conn, 400)["error"] ==
"Metric `conversion_rate` can only be queried with event:goal filters or dimensions"
end
test "validates that dimensions are valid", %{conn: conn, site: site} do
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visitors"],
"date_range" => "all",
"dimensions" => ["badproperty"]
})
assert json_response(conn, 400)["error"] =~ "Invalid dimensions"
end
test "empty custom property is invalid", %{conn: conn, site: site} do
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visitors"],
"date_range" => "all",
"dimensions" => ["event:props:"]
})
assert json_response(conn, 400)["error"] =~ "Invalid dimensions"
end
test "validates that correct date range is used", %{conn: conn, site: site} do
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visitors"],
"date_range" => "bad_period",
"dimensions" => ["event:name"]
})
assert json_response(conn, 400)["error"] =~ "Invalid date_range"
end
test "fails when an invalid metric is provided", %{conn: conn, site: site} do
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visitors", "baa"],
"date_range" => "all",
"dimensions" => ["event:name"]
})
assert json_response(conn, 400)["error"] =~ "Unknown metric '\"baa\"'"
end
test "session metrics cannot be used with event:name dimension", %{conn: conn, site: site} do
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visitors", "bounce_rate"],
"date_range" => "all",
"dimensions" => ["event:name"]
})
assert json_response(conn, 400)["error"] =~
"Session metric(s) `bounce_rate` cannot be queried along with event dimensions"
end
test "session metrics cannot be used with event:props:* dimension", %{conn: conn, site: site} do
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"metrics" => ["visitors", "bounce_rate"],
"date_range" => "all",
"dimensions" => ["event:props:url"]
})
assert json_response(conn, 400)["error"] =~
"Session metric(s) `bounce_rate` cannot be queried along with event dimensions"
end
test "validates that metric views_per_visit cannot be used with event:page filter", %{
conn: conn,
site: site
} do
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"date_range" => "all",
"metrics" => ["views_per_visit"],
"filters" => [["is", "event:page", ["/something"]]]
})
assert json_response(conn, 400) == %{
"error" =>
"Metric `views_per_visit` cannot be queried with a filter on `event:page`"
}
end
test "validates that metric views_per_visit cannot be used together with dimensions", %{
conn: conn,
site: site
} do
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"date_range" => "all",
"metrics" => ["views_per_visit"],
"dimensions" => ["event:name"]
})
assert json_response(conn, 400) == %{
"error" => "Metric `views_per_visit` cannot be queried with `dimensions`"
}
end
test "validates a metric can't be asked multiple times", %{
conn: conn,
site: site
} do
conn =
post(conn, "/api/v2/query", %{
"site_id" => site.domain,
"date_range" => "all",
"metrics" => ["views_per_visit", "visitors", "visitors"]
})
assert json_response(conn, 400) == %{
"error" => "Metrics cannot be queried multiple times"
}
end
end
end