plausible/analytics
1
1
Fork 0
mirror of https://github.com/plausible/analytics.git synced 2024-09-11 09:56:02 +03:00
Code Issues Packages Projects Releases Wiki Activity

Conditionally support switching between v1 and v2 clickhouse schemas (#2780)

Browse Source 
* Remove ClickhouseSetup module

This has been an implicit point of contact to many
tests. From now on the goal is for each test to maintain
its own, isolated setup so that no accidental clashes
and implicit assumptions are relied upon.

* Implement v2 schema check

An environment variable V2_MIGRATION_DONE acts like
a feature flag, switching plausible from using old events/sessions
schemas to v2 schemas introduced by NumericIDs migration.

* Run both test suites sequentially

While the code for v1 and v2 schemas must be kept still,
we will from now on run tests against both code paths.
Secondary test run will set V2_MIGRATION_DONE=1 variable,
thus making all `Plausible.v2?()` checks return `true'.

* Remove unused function

This is a remnant from the short period when
we would check for existing events before allowing
creating a new site.

* Update test setups/factories with v2 migration check

* Make GateKeeper return site id along with :allow

* Make Billing module check for v2 schema

* Make ingestion aware of v2 schema

* Disable site transfers for when v2 is live

In a separate changeset we will implement simplified
site transfer for when v2 migration is complete.
The new transfer will only rename the site domain in postgres
and keep track of the original site prior to the transfer
so we keep an ingestion grace period until the customers
redeploy their scripting.

* Make Stats base queries aware of v2 schema switch

* Update breakdown with v2 conditionals

* Update pageview local start with v2 check

* Update current visitoris with v2 check

* Update stats controller with v2 checks

* Update external controller with v2 checks

* Update remaining tests with proper fixtures

* Rewrite redundant assignment

* Remove unused alias

* Mute credo, this is not the right time

* Add test_helper prompt

* Fetch priv dir so it works with a release

* Fetch distinct partitions only

* Don't limit inspect output for partitions

* Ensure SQL is printed to IO

* Remove redundant domain fixture
This commit is contained in:
hq1 2023-03-27 13:52:42 +02:00 committed by GitHub
parent a75d0b35b0
commit d2f2c69387
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
43 changed files with 1211 additions and 1185 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/elixir.yml vendored
View File

@ -70,8 +70,12 @@ jobs:
run: npm run deploy --prefix ./tracker
- name: Check Credo Warnings
run: mix credo diff --from-git-merge-base origin/master
- name: Run tests
- name: Run tests (against v1 schema)
run: mix test --include slow --max-failures 1 --warnings-as-errors
- name: Run tests (against v2 schema)
run: mix test --include slow --max-failures 1 --warnings-as-errors
env:
V2_MIGRATION_DONE: 1
- name: Check Dialyzer
run: mix dialyzer
env:

4
config/runtime.exs
View File

@ -99,6 +99,8 @@ ch_db_url =
|> get_var_from_path_or_env("CLICKHOUSE_MAX_BUFFER_SIZE", "10000")
|> Integer.parse()
v2_migration_done = get_var_from_path_or_env(config_dir, "V2_MIGRATION_DONE")
### Mandatory params End
build_metadata_raw = get_var_from_path_or_env(config_dir, "BUILD_METADATA", "{}")
@ -131,6 +133,8 @@ runtime_metadata = [
config :plausible, :runtime_metadata, runtime_metadata
config :plausible, :v2_migration_done, v2_migration_done
sentry_dsn = get_var_from_path_or_env(config_dir, "SENTRY_DSN")
honeycomb_api_key = get_var_from_path_or_env(config_dir, "HONEYCOMB_API_KEY")
honeycomb_dataset = get_var_from_path_or_env(config_dir, "HONEYCOMB_DATASET")

5
lib/plausible.ex
View File

@ -6,4 +6,9 @@ defmodule Plausible do
Contexts are also responsible for managing your data, regardless
if it comes from the database, an external API or others.
"""
@spec v2?() :: boolean()
def v2?() do
Plausible.DataMigration.NumericIDs.ready?()
end
end

46
lib/plausible/billing/billing.ex
View File

@ -113,17 +113,36 @@ defmodule Plausible.Billing do
def last_two_billing_months_usage(user, today \\ Timex.today()) do
{first, second} = last_two_billing_cycles(user, today)
domains = Plausible.Sites.owned_sites_domains(user)
usage_for_sites = fn domains, date_range ->
{pageviews, custom_events} = Plausible.Stats.Clickhouse.usage_breakdown(domains, date_range)
pageviews + custom_events
if Plausible.v2?() do
site_ids = Plausible.Sites.owned_site_ids(user)
usage_for_sites = fn site_ids, date_range ->
{pageviews, custom_events} =
Plausible.Stats.Clickhouse.usage_breakdown(site_ids, date_range)
pageviews + custom_events
end
{
usage_for_sites.(site_ids, first),
usage_for_sites.(site_ids, second)
}
else
domains = Plausible.Sites.owned_sites_domains(user)
usage_for_sites = fn domains, date_range ->
{pageviews, custom_events} =
Plausible.Stats.Clickhouse.usage_breakdown(domains, date_range)
pageviews + custom_events
end
{
usage_for_sites.(domains, first),
usage_for_sites.(domains, second)
}
end
{
usage_for_sites.(domains, first),
usage_for_sites.(domains, second)
}
end
def last_two_billing_cycles(user, today \\ Timex.today()) do
@ -147,8 +166,13 @@ defmodule Plausible.Billing do
end
def usage_breakdown(user) do
domains = Plausible.Sites.owned_sites_domains(user)
Plausible.Stats.Clickhouse.usage_breakdown(domains)
if Plausible.v2?() do
site_ids = Plausible.Sites.owned_site_ids(user)
Plausible.Stats.Clickhouse.usage_breakdown(site_ids)
else
domains = Plausible.Sites.owned_sites_domains(user)
Plausible.Stats.Clickhouse.usage_breakdown(domains)
end
end
@doc """

6
lib/plausible/data_migration.ex
View File

@ -13,7 +13,7 @@ defmodule Plausible.DataMigration do
@repo repo
def run_sql_confirm(name, assigns \\ []) do
query = unwrap(name, assigns)
query = unwrap_with_io(name, assigns)
confirm("Execute?", fn -> do_run(name, query) end)
end
@ -30,7 +30,9 @@ defmodule Plausible.DataMigration do
end
defp unwrap(name, assigns) do
"priv/data_migrations"
:plausible
|> :code.priv_dir()
|> Path.join("data_migrations")
|> Path.join(@dir)
|> Path.join("sql")
|> Path.join(name <> ".sql.eex")

6
lib/plausible/data_migration/numeric_ids.ex
View File

@ -7,6 +7,10 @@ defmodule Plausible.DataMigration.NumericIDs do
@table_settings "SETTINGS index_granularity = 8192, storage_policy = 'tiered'"
def ready?() do
Application.get_env(:plausible, :v2_migration_done) || false
end
# credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity
def run(opts \\ []) do
interactive? = Keyword.get(opts, :interactive?, true)
@ -65,7 +69,7 @@ defmodule Plausible.DataMigration.NumericIDs do
- table_settings: #{table_settings}
- db url: #{db_url}
- cluster?: #{cluster?}
- partitions to do: #{inspect(partitions, pretty: true, width: 80)}
- partitions to do: #{inspect(partitions, pretty: true, limit: :infinity, width: 80)}
- start from: #{start_from}
- stop at: #{stop_at}
""")

7
lib/plausible/event/write_buffer.ex
View File

@ -64,7 +64,12 @@ defmodule Plausible.Event.WriteBuffer do
events ->
Logger.info("Flushing #{length(events)} events")
events = Enum.map(events, &(Map.from_struct(&1) |> Map.delete(:__meta__)))
IngestRepo.insert_all(Plausible.ClickhouseEvent, events)
if Plausible.v2?() do
IngestRepo.insert_all(Plausible.ClickhouseEventV2, events)
else
IngestRepo.insert_all(Plausible.ClickhouseEvent, events)
end
end
end

26
lib/plausible/ingestion/event.ex
View File

@ -7,9 +7,11 @@ defmodule Plausible.Ingestion.Event do
"""
alias Plausible.Ingestion.Request
alias Plausible.ClickhouseEvent
alias Plausible.ClickhouseEventV2
alias Plausible.Site.GateKeeper
defstruct domain: nil,
site_id: nil,
clickhouse_event_attrs: %{},
clickhouse_event: nil,
dropped?: false,
@ -26,8 +28,9 @@ defmodule Plausible.Ingestion.Event do
@type t() :: %__MODULE__{
domain: String.t() | nil,
site_id: pos_integer() | nil,
clickhouse_event_attrs: map(),
clickhouse_event: %ClickhouseEvent{} | nil,
clickhouse_event: %ClickhouseEvent{} | %ClickhouseEventV2{} | nil,
dropped?: boolean(),
drop_reason: drop_reason(),
request: Request.t(),
@ -43,10 +46,10 @@ defmodule Plausible.Ingestion.Event do
else
Enum.reduce(domains, [], fn domain, acc ->
case GateKeeper.check(domain) do
:allow ->
{:allow, site_id} ->
processed =
domain
|> new(request)
|> new(site_id, request)
|> process_unless_dropped(pipeline())
[processed | acc]
@ -117,6 +120,10 @@ defmodule Plausible.Ingestion.Event do
struct!(__MODULE__, domain: domain, request: request)
end
defp new(domain, site_id, request) do
struct!(__MODULE__, domain: domain, site_id: site_id, request: request)
end
defp drop(%__MODULE__{} = event, reason, attrs \\ []) do
fields =
attrs
@ -156,6 +163,7 @@ defmodule Plausible.Ingestion.Event do
defp put_basic_info(%__MODULE__{} = event) do
update_attrs(event, %{
domain: event.domain,
site_id: event.site_id,
timestamp: event.request.timestamp,
name: event.request.event_name,
hostname: event.request.hostname,
@ -217,9 +225,15 @@ defmodule Plausible.Ingestion.Event do
defp validate_clickhouse_event(%__MODULE__{} = event) do
clickhouse_event =
event
|> Map.fetch!(:clickhouse_event_attrs)
|> ClickhouseEvent.new()
if Plausible.v2?() do
event
|> Map.fetch!(:clickhouse_event_attrs)
|> ClickhouseEventV2.new()
else
event
|> Map.fetch!(:clickhouse_event_attrs)
|> ClickhouseEvent.new()
end
case Ecto.Changeset.apply_action(clickhouse_event, nil) do
{:ok, valid_clickhouse_event} ->

119
lib/plausible/session/cache_store.ex
View File

@ -22,7 +22,14 @@ defmodule Plausible.Session.CacheStore do
defp find_session(_domain, nil), do: nil
defp find_session(event, user_id) do
case Cachex.get(:sessions, {event.domain, user_id}) do
from_cache =
if Plausible.v2?() do
Cachex.get(:sessions, {event.site_id, user_id})
else
Cachex.get(:sessions, {event.domain, user_id})
end
case from_cache do
{:ok, nil} ->
nil
@ -38,7 +45,13 @@ defmodule Plausible.Session.CacheStore do
end
defp persist_session(session) do
key = {session.domain, session.user_id}
key =
if Plausible.v2?() do
{session.site_id, session.user_id}
else
{session.domain, session.user_id}
end
Cachex.put(:sessions, key, session, ttl: :timer.minutes(30))
session
end
@ -68,38 +81,74 @@ defmodule Plausible.Session.CacheStore do
end
defp new_session_from_event(event) do
%Plausible.ClickhouseSession{
sign: 1,
session_id: Plausible.ClickhouseSession.random_uint64(),
hostname: event.hostname,
domain: event.domain,
user_id: event.user_id,
entry_page: event.pathname,
exit_page: event.pathname,
is_bounce: true,
duration: 0,
pageviews: if(event.name == "pageview", do: 1, else: 0),
events: 1,
referrer: event.referrer,
referrer_source: event.referrer_source,
utm_medium: event.utm_medium,
utm_source: event.utm_source,
utm_campaign: event.utm_campaign,
utm_content: event.utm_content,
utm_term: event.utm_term,
country_code: event.country_code,
subdivision1_code: event.subdivision1_code,
subdivision2_code: event.subdivision2_code,
city_geoname_id: event.city_geoname_id,
screen_size: event.screen_size,
operating_system: event.operating_system,
operating_system_version: event.operating_system_version,
browser: event.browser,
browser_version: event.browser_version,
timestamp: event.timestamp,
start: event.timestamp,
"entry_meta.key": Map.get(event, :"meta.key"),
"entry_meta.value": Map.get(event, :"meta.value")
}
if Plausible.v2?() do
%Plausible.ClickhouseSessionV2{
sign: 1,
session_id: Plausible.ClickhouseSession.random_uint64(),
hostname: event.hostname,
site_id: event.site_id,
user_id: event.user_id,
entry_page: event.pathname,
exit_page: event.pathname,
is_bounce: true,
duration: 0,
pageviews: if(event.name == "pageview", do: 1, else: 0),
events: 1,
referrer: event.referrer,
referrer_source: event.referrer_source,
utm_medium: event.utm_medium,
utm_source: event.utm_source,
utm_campaign: event.utm_campaign,
utm_content: event.utm_content,
utm_term: event.utm_term,
country_code: event.country_code,
subdivision1_code: event.subdivision1_code,
subdivision2_code: event.subdivision2_code,
city_geoname_id: event.city_geoname_id,
screen_size: event.screen_size,
operating_system: event.operating_system,
operating_system_version: event.operating_system_version,
browser: event.browser,
browser_version: event.browser_version,
timestamp: event.timestamp,
start: event.timestamp,
"entry_meta.key": Map.get(event, :"meta.key"),
"entry_meta.value": Map.get(event, :"meta.value")
}
else
%Plausible.ClickhouseSession{
sign: 1,
session_id: Plausible.ClickhouseSession.random_uint64(),
hostname: event.hostname,
domain: event.domain,
user_id: event.user_id,
entry_page: event.pathname,
exit_page: event.pathname,
is_bounce: true,
duration: 0,
pageviews: if(event.name == "pageview", do: 1, else: 0),
events: 1,
referrer: event.referrer,
referrer_source: event.referrer_source,
utm_medium: event.utm_medium,
utm_source: event.utm_source,
utm_campaign: event.utm_campaign,
utm_content: event.utm_content,
utm_term: event.utm_term,
country_code: event.country_code,
subdivision1_code: event.subdivision1_code,
subdivision2_code: event.subdivision2_code,
city_geoname_id: event.city_geoname_id,
screen_size: event.screen_size,
operating_system: event.operating_system,
operating_system_version: event.operating_system_version,
browser: event.browser,
browser_version: event.browser_version,
timestamp: event.timestamp,
start: event.timestamp,
"entry_meta.key": Map.get(event, :"meta.key"),
"entry_meta.value": Map.get(event, :"meta.value")
}
end
end
end

6
lib/plausible/session/write_buffer.ex
View File

@ -69,7 +69,11 @@ defmodule Plausible.Session.WriteBuffer do
|> Enum.map(&(Map.from_struct(&1) |> Map.delete(:__meta__)))
|> Enum.reverse()
IngestRepo.insert_all(Plausible.ClickhouseSession, sessions)
if Plausible.v2?() do
IngestRepo.insert_all(Plausible.ClickhouseSessionV2, sessions)
else
IngestRepo.insert_all(Plausible.ClickhouseSession, sessions)
end
end
end

44
lib/plausible/site/admin.ex
View File

@ -84,26 +84,32 @@ defmodule Plausible.SiteAdmin do
end
def transfer_data([from_site], params) do
to_site = Repo.get_by(Plausible.Site, domain: params["domain"])
if to_site do
event_q = event_transfer_query(from_site.domain, to_site.domain)
{:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, event_q, [], timeout: 30_000)
session_q = session_transfer_query(from_site.domain, to_site.domain)
{:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, session_q, [], timeout: 30_000)
start_date = Plausible.Stats.Clickhouse.pageview_start_date_local(from_site)
{:ok, _} =
to_site
|> Plausible.Site.set_stats_start_date(start_date)
|> Plausible.Site.set_native_stats_start_at(from_site.native_stats_start_at)
|> Repo.update()
:ok
if Plausible.v2?() do
{:error, "Transfers temporarily unspported with v2"}
else
{:error, "Cannot transfer to non-existing domain"}
to_site = Repo.get_by(Plausible.Site, domain: params["domain"])
if to_site do
event_q = event_transfer_query(from_site.domain, to_site.domain)
{:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, event_q, [], timeout: 30_000)
session_q = session_transfer_query(from_site.domain, to_site.domain)
{:ok, _} =
Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, session_q, [], timeout: 30_000)
start_date = Plausible.Stats.Clickhouse.pageview_start_date_local(from_site)
{:ok, _} =
to_site
|> Plausible.Site.set_stats_start_date(start_date)
|> Plausible.Site.set_native_stats_start_at(from_site.native_stats_start_at)
|> Repo.update()
:ok
else
{:error, "Cannot transfer to non-existing domain"}
end
end
end

20
lib/plausible/site/gate_keeper.ex
View File

@ -1,9 +1,9 @@
defmodule Plausible.Site.GateKeeper do
@type policy() :: :allow | :not_found | :block | :throttle
@type site_id() :: pos_integer()
@policy_for_non_existing_sites :not_found
@policy_on_rate_limiting_backend_error :allow
@type t() :: :allow | {:deny, policy()}
@type t() :: {:allow, site_id()} | {:deny, policy()}
@moduledoc """
Thin wrapper around Hammer for gate keeping domain-specific events
@ -23,7 +23,7 @@ defmodule Plausible.Site.GateKeeper do
The module defines two policies outside the regular bucket inspection:
* when the the site is not found in cache: #{@policy_for_non_existing_sites}
* when the underlying rate limiting mechanism returns
an internal error: #{@policy_on_rate_limiting_backend_error}
an internal error: :allow
"""
alias Plausible.Site
alias Plausible.Site.Cache
@ -33,7 +33,7 @@ defmodule Plausible.Site.GateKeeper do
@spec check(String.t(), Keyword.t()) :: t()
def check(domain, opts \\ []) when is_binary(domain) do
case policy(domain, opts) do
:allow -> :allow
{:allow, site_id} -> {:allow, site_id}
other -> {:deny, other}
end
end
@ -56,15 +56,15 @@ defmodule Plausible.Site.GateKeeper do
result
end
defp check_rate_limit(%Site{ingest_rate_limit_threshold: nil}, _opts) do
:allow
defp check_rate_limit(%Site{id: site_id, ingest_rate_limit_threshold: nil}, _opts) do
{:allow, site_id}
end
defp check_rate_limit(%Site{ingest_rate_limit_threshold: 0}, _opts) do
:block
end
defp check_rate_limit(%Site{ingest_rate_limit_threshold: threshold} = site, opts)
defp check_rate_limit(%Site{id: site_id, ingest_rate_limit_threshold: threshold} = site, opts)
when is_integer(threshold) do
key = Keyword.get(opts, :key, key(site.domain))
scale_ms = site.ingest_rate_limit_scale_seconds * 1_000
@ -74,14 +74,14 @@ defmodule Plausible.Site.GateKeeper do
:throttle
{:allow, _} ->
:allow
{:allow, site_id}
{:error, reason} ->
Logger.error(
"Error checking rate limit for '#{key}': #{inspect(reason)}. Falling back to: #{@policy_on_rate_limiting_backend_error}"
"Error checking rate limit for '#{key}': #{inspect(reason)}. Falling back to: :allow"
)
@policy_on_rate_limiting_backend_error
{:allow, site_id}
end
end
end

13
lib/plausible/sites.ex
View File

@ -1,6 +1,5 @@
defmodule Plausible.Sites do
use Plausible.Repo
alias Plausible.ClickhouseRepo
alias Plausible.Site
alias Plausible.Site.SharedLink
import Ecto.Query
@ -77,11 +76,6 @@ defmodule Plausible.Sites do
!!stats_start_date(site)
end
def has_events?(domain) do
q = from e in "events", where: e.domain == ^domain, select: true, limit: 1
ClickhouseRepo.one(q) == true
end
def create_shared_link(site, name, password \\ nil) do
changes =
SharedLink.changeset(
@ -158,6 +152,13 @@ defmodule Plausible.Sites do
|> Repo.all()
end
def owned_site_ids(user) do
user
|> owned_sites_query()
|> select([site], site.id)
|> Repo.all()
end
defp owned_sites_query(user) do
from(s in Site,
join: sm in Site.Membership,

36
lib/plausible/stats/base.ex
View File

@ -35,11 +35,19 @@ defmodule Plausible.Stats.Base do
{first_datetime, last_datetime} = utc_boundaries(query, site)
q =
from(
e in "events",
where: e.domain == ^site.domain,
where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
)
if Plausible.v2?() do
from(
e in "events_v2",
where: e.site_id == ^site.id,
where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
)
else
from(
e in "events",
where: e.domain == ^site.domain,
where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
)
end
|> add_sample_hint(query)
q =
@ -190,11 +198,19 @@ defmodule Plausible.Stats.Base do
{first_datetime, last_datetime} = utc_boundaries(query, site)
sessions_q =
from(
s in "sessions",
where: s.domain == ^site.domain,
where: s.start >= ^first_datetime and s.start < ^last_datetime
)
if Plausible.v2?() do
from(
s in "sessions_v2",
where: s.site_id == ^site.id,
where: s.start >= ^first_datetime and s.start < ^last_datetime
)
else
from(
s in "sessions",
where: s.domain == ^site.domain,
where: s.start >= ^first_datetime and s.start < ^last_datetime
)
end
|> add_sample_hint(query)
|> filter_by_entry_props(query)

8
lib/plausible/stats/breakdown.ex
View File

@ -312,7 +312,7 @@ defmodule Plausible.Stats.Breakdown do
end
defp do_group_by(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events", _}}} = q,
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,
"event:props:" <> prop
) do
q =
@ -336,7 +336,7 @@ defmodule Plausible.Stats.Breakdown do
end
defp do_group_by(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events", _}}} = q,
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,
"event:name"
) do
from(
@ -348,7 +348,7 @@ defmodule Plausible.Stats.Breakdown do
end
defp do_group_by(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events", _}}} = q,
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,
"event:page"
) do
from(
@ -360,7 +360,7 @@ defmodule Plausible.Stats.Breakdown do
end
defp do_group_by(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events", _}}} = q,
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,
"event:page_match"
) do
case Map.get(q, :__private_match_sources__) do

162
lib/plausible/stats/clickhouse.ex
View File

@ -8,12 +8,23 @@ defmodule Plausible.Stats.Clickhouse do
@spec pageview_start_date_local(Plausible.Site.t()) :: Date.t() | nil
def pageview_start_date_local(site) do
datetime =
ClickhouseRepo.one(
from e in "events",
select: fragment("min(?)", e.timestamp),
where: e.domain == ^site.domain,
where: e.timestamp >= ^site.native_stats_start_at
)
if Plausible.v2?() do
ClickhouseRepo.one(
from(e in "events_v2",
select: fragment("min(?)", e.timestamp),
where: e.site_id == ^site.id,
where: e.timestamp >= ^site.native_stats_start_at
)
)
else
ClickhouseRepo.one(
from(e in "events",
select: fragment("min(?)", e.timestamp),
where: e.domain == ^site.domain,
where: e.timestamp >= ^site.native_stats_start_at
)
)
end
case datetime do
# no stats for this domain yet
@ -29,28 +40,29 @@ defmodule Plausible.Stats.Clickhouse do
def imported_pageview_count(site) do
Plausible.ClickhouseRepo.one(
from i in "imported_visitors",
from(i in "imported_visitors",
where: i.site_id == ^site.id,
select: sum(i.pageviews)
)
)
end
def usage_breakdown(domains) do
def usage_breakdown(domains_or_site_ids) do
range =
Date.range(
Timex.shift(Timex.today(), days: -30),
Timex.today()
)
usage_breakdown(domains, range)
usage_breakdown(domains_or_site_ids, range)
end
def usage_breakdown(domains, date_range) do
def usage_breakdown([d | _] = domains, date_range) when is_binary(d) do
Enum.chunk_every(domains, 300)
|> Enum.reduce({0, 0}, fn domains, {pageviews_total, custom_events_total} ->
{chunk_pageviews, chunk_custom_events} =
ClickhouseRepo.one(
from e in "events",
from(e in "events",
where: e.domain in ^domains,
where: fragment("toDate(?)", e.timestamp) >= ^date_range.first,
where: fragment("toDate(?)", e.timestamp) <= ^date_range.last,
@ -58,12 +70,35 @@ defmodule Plausible.Stats.Clickhouse do
fragment("countIf(? = 'pageview')", e.name),
fragment("countIf(? != 'pageview')", e.name)
}
)
)
{pageviews_total + chunk_pageviews, custom_events_total + chunk_custom_events}
end)
end
def usage_breakdown([sid | _] = site_ids, date_range) when is_integer(sid) do
Enum.chunk_every(site_ids, 300)
|> Enum.reduce({0, 0}, fn site_ids, {pageviews_total, custom_events_total} ->
{chunk_pageviews, chunk_custom_events} =
ClickhouseRepo.one(
from(e in "events_v2",
where: e.site_id in ^site_ids,
where: fragment("toDate(?)", e.timestamp) >= ^date_range.first,
where: fragment("toDate(?)", e.timestamp) <= ^date_range.last,
select: {
fragment("countIf(? = 'pageview')", e.name),
fragment("countIf(? != 'pageview')", e.name)
}
)
)
{pageviews_total + chunk_pageviews, custom_events_total + chunk_custom_events}
end)
end
def usage_breakdown([], _date_range), do: {0, 0}
def top_sources(site, query, limit, page, show_noref \\ false, include_details) do
offset = (page - 1) * limit
@ -151,47 +186,87 @@ defmodule Plausible.Stats.Clickhouse do
def current_visitors(site, query) do
Plausible.ClickhouseRepo.one(
from e in base_query(site, query),
from(e in base_query(site, query),
select: uniq(e.user_id)
)
)
end
def has_pageviews?(site) do
ClickhouseRepo.exists?(
from(e in "events",
where:
e.domain == ^site.domain and
e.name == "pageview" and
e.timestamp >=
^site.native_stats_start_at
if Plausible.v2?() do
ClickhouseRepo.exists?(
from(e in "events_v2",
where:
e.site_id == ^site.id and
e.name == "pageview" and
e.timestamp >=
^site.native_stats_start_at
)
)
)
else
ClickhouseRepo.exists?(
from(e in "events",
where:
e.domain == ^site.domain and
e.name == "pageview" and
e.timestamp >=
^site.native_stats_start_at
)
)
end
end
def last_24h_visitors([]), do: %{}
def last_24h_visitors(sites) do
domains = Enum.map(sites, & &1.domain)
if Plausible.v2?() do
site_id_to_domain_mapping = for site <- sites, do: {site.id, site.domain}, into: %{}
ClickhouseRepo.all(
from e in "events",
group_by: e.domain,
where: e.domain in ^domains,
where: e.timestamp > fragment("now() - INTERVAL 24 HOUR"),
select: {e.domain, fragment("uniq(user_id)")}
)
|> Enum.into(%{})
ClickhouseRepo.all(
from(e in "events_v2",
group_by: e.site_id,
where: e.site_id in ^Map.keys(site_id_to_domain_mapping),
where: e.timestamp > fragment("now() - INTERVAL 24 HOUR"),
select: {e.site_id, fragment("uniq(user_id)")}
)
)
|> Enum.map(fn {site_id, user_id} ->
{site_id_to_domain_mapping[site_id], user_id}
end)
|> Enum.into(%{})
else
domains = Enum.map(sites, & &1.domain)
ClickhouseRepo.all(
from(e in "events",
group_by: e.domain,
where: e.domain in ^domains,
where: e.timestamp > fragment("now() - INTERVAL 24 HOUR"),
select: {e.domain, fragment("uniq(user_id)")}
)
)
|> Enum.into(%{})
end
end
# credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity
defp base_session_query(site, query) do
{first_datetime, last_datetime} = utc_boundaries(query, site)
q =
from(s in "sessions",
hints: ["SAMPLE 10000000"],
where: s.domain == ^site.domain,
where: s.timestamp >= ^first_datetime and s.start < ^last_datetime
)
if Plausible.v2?() do
from(s in "sessions_v2",
hints: ["SAMPLE 10000000"],
where: s.site_id == ^site.id,
where: s.timestamp >= ^first_datetime and s.start < ^last_datetime
)
else
from(s in "sessions",
hints: ["SAMPLE 10000000"],
where: s.domain == ^site.domain,
where: s.timestamp >= ^first_datetime and s.start < ^last_datetime
)
end
q =
if query.filters["source"] do
@ -302,15 +377,24 @@ defmodule Plausible.Stats.Clickhouse do
end
end
# credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity
defp base_query_bare(site, query) do
{first_datetime, last_datetime} = utc_boundaries(query, site)
q =
from(e in "events",
hints: ["SAMPLE 10000000"],
where: e.domain == ^site.domain,
where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
)
if Plausible.v2?() do
from(e in "events_v2",
hints: ["SAMPLE 10000000"],
where: e.site_id == ^site.id,
where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
)
else
from(e in "events",
hints: ["SAMPLE 10000000"],
where: e.domain == ^site.domain,
where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
)
end
q =
if query.filters["screen"] do

21
lib/plausible/stats/current_visitors.ex
View File

@ -8,11 +8,20 @@ defmodule Plausible.Stats.CurrentVisitors do
|> Timex.shift(minutes: -5)
|> NaiveDateTime.truncate(:second)
ClickhouseRepo.one(
from e in "events",
where: e.domain == ^site.domain,
where: e.timestamp >= ^first_datetime,
select: uniq(e.user_id)
)
if Plausible.v2?() do
ClickhouseRepo.one(
from e in "events_v2",
where: e.site_id == ^site.id,
where: e.timestamp >= ^first_datetime,
select: uniq(e.user_id)
)
else
ClickhouseRepo.one(
from e in "events",
where: e.domain == ^site.domain,
where: e.timestamp >= ^first_datetime,
select: uniq(e.user_id)
)
end
end
end

9
lib/plausible_web/controllers/api/stats_controller.ex
View File

@ -1161,9 +1161,14 @@ defmodule PlausibleWeb.Api.StatsController do
site = conn.assigns[:site]
with :ok <- validate_params(params) do
query = Query.from(site, params) |> Filters.add_prefix()
query =
Query.from(site, params)
|> Filters.add_prefix()
json(conn, Stats.filter_suggestions(site, query, params["filter_name"], params["q"]))
json(
conn,
Stats.filter_suggestions(site, query, params["filter_name"], params["q"])
)
else
{:error, message} when is_binary(message) -> bad_request(conn, message)
end

2
mix.lock
View File

@ -10,7 +10,7 @@
"cachex": {:hex, :cachex, "3.4.0", "868b2959ea4aeb328c6b60ff66c8d5123c083466ad3c33d3d8b5f142e13101fb", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "370123b1ab4fba4d2965fb18f87fd758325709787c8c5fce35b3fe80645ccbe5"},
"castore": {:hex, :castore, "0.1.22", "4127549e411bedd012ca3a308dede574f43819fe9394254ca55ab4895abfa1a2", [:mix], [], "hexpm", "c17576df47eb5aa1ee40cc4134316a99f5cad3e215d5c77b8dd3cfef12a22cac"},
"certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"},
"ch": {:git, "https://github.com/ruslandoga/ch.git", "5894f81e246c049b9db5186497189cce226947aa", []},
"ch": {:git, "https://github.com/plausible/ch.git", "f411daa07c6310d3308f81397905df65330aeb64", []},
"chatterbox": {:hex, :ts_chatterbox, "0.13.0", "6f059d97bcaa758b8ea6fffe2b3b81362bd06b639d3ea2bb088335511d691ebf", [:rebar3], [{:hpack, "~> 0.2.3", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "b93d19104d86af0b3f2566c4cba2a57d2e06d103728246ba1ac6c3c0ff010aa7"},
"chto": {:git, "https://github.com/plausible/chto.git", "316421ede947541bac32a853d70b0ca8ba003220", []},
"combination": {:hex, :combination, "0.0.3", "746aedca63d833293ec6e835aa1f34974868829b1486b1e1cb0685f0b2ae1f41", [:mix], [], "hexpm", "72b099f463df42ef7dc6371d250c7070b57b6c5902853f69deb894f79eda18ca"},

2
priv/data_migrations/NumericIDs/sql/list-partitions.sql.eex
View File

@ -1,4 +1,4 @@
SELECT partition
SELECT distinct(partition)
FROM system.parts
WHERE table = 'events'
<%= if @start_from do %>AND partition >= '<%= @start_from %>'<% end %>

6
test/plausible/auth/auth_test.exs
View File

@ -18,7 +18,11 @@ defmodule Plausible.AuthTest do
test "is true if user does have events" do
user = insert(:user)
insert(:site, members: [user], domain: "test-site.com")
site = insert(:site, members: [user])
populate_stats(site, [
build(:pageview)
])
assert Auth.has_active_sites?(user)
end

16
test/plausible/billing/billing_test.exs
View File

@ -133,10 +133,10 @@ defmodule Plausible.BillingTest do
site = insert(:site, members: [user])
create_pageviews([
%{domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]},
%{domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]},
%{domain: site.domain, timestamp: ~N[2020-11-01 00:00:00]},
%{domain: site.domain, timestamp: ~N[2020-10-31 00:00:00]}
%{site: site, timestamp: ~N[2021-01-01 00:00:00]},
%{site: site, timestamp: ~N[2020-12-31 00:00:00]},
%{site: site, timestamp: ~N[2020-11-01 00:00:00]},
%{site: site, timestamp: ~N[2020-10-31 00:00:00]}
])
assert Billing.last_two_billing_months_usage(user, today) == {1, 1}
@ -163,10 +163,10 @@ defmodule Plausible.BillingTest do
)
create_pageviews([
%{domain: owner_site.domain, timestamp: ~N[2020-12-31 00:00:00]},
%{domain: admin_site.domain, timestamp: ~N[2020-12-31 00:00:00]},
%{domain: owner_site.domain, timestamp: ~N[2020-11-01 00:00:00]},
%{domain: admin_site.domain, timestamp: ~N[2020-11-01 00:00:00]}
%{site: owner_site, timestamp: ~N[2020-12-31 00:00:00]},
%{site: admin_site, timestamp: ~N[2020-12-31 00:00:00]},
%{site: owner_site, timestamp: ~N[2020-11-01 00:00:00]},
%{site: admin_site, timestamp: ~N[2020-11-01 00:00:00]}
])
assert Billing.last_two_billing_months_usage(user, today) == {1, 1}

27
test/plausible/session/cache_store_test.exs
View File

@ -37,7 +37,13 @@ defmodule Plausible.Session.CacheStoreTest do
assert_receive({WriteBuffer, :insert, [sessions]})
assert [session] = sessions
assert session.hostname == event.hostname
assert session.domain == event.domain
if Plausible.v2?() do
assert session.site_id == event.site_id
else
assert session.domain == event.domain
end
assert session.user_id == event.user_id
assert session.entry_page == event.pathname
assert session.exit_page == event.pathname
@ -68,12 +74,9 @@ defmodule Plausible.Session.CacheStoreTest do
timestamp = Timex.now()
event1 = build(:event, name: "pageview", timestamp: timestamp |> Timex.shift(seconds: -10))
event2 =
build(:event,
domain: event1.domain,
user_id: event1.user_id,
name: "pageview",
timestamp: timestamp,
event2 = %{
event1
| timestamp: timestamp,
country_code: "US",
subdivision1_code: "SUB1",
subdivision2_code: "SUB2",
@ -83,7 +86,7 @@ defmodule Plausible.Session.CacheStoreTest do
operating_system_version: "11",
browser: "Firefox",
browser_version: "10"
)
}
CacheStore.on_event(event1, nil, buffer)
CacheStore.on_event(event2, nil, buffer)
@ -107,13 +110,7 @@ defmodule Plausible.Session.CacheStoreTest do
timestamp = Timex.now()
event1 = build(:event, name: "pageview", timestamp: timestamp |> Timex.shift(seconds: 10))
event2 =
build(:event,
domain: event1.domain,
user_id: event1.user_id,
name: "pageview",
timestamp: timestamp
)
event2 = %{event1 | timestamp: timestamp}
CacheStore.on_event(event1, nil, buffer)
CacheStore.on_event(event2, nil, buffer)

106
test/plausible/site/admin_test.exs
View File

@ -1,76 +1,98 @@
defmodule Plausible.SiteAdminTest do
use Plausible.DataCase, async: true
alias Plausible.{SiteAdmin, ClickhouseRepo, ClickhouseEvent, ClickhouseSession}
alias Plausible.{
SiteAdmin,
ClickhouseRepo,
ClickhouseEvent,
ClickhouseSession
}
describe "transfer_data" do
test "event and session structs remain the same after transfer" do
from_site = insert(:site)
to_site = insert(:site)
if Plausible.v2?() do
:ok
else
from_site = insert(:site)
to_site = insert(:site)
populate_stats(from_site, [build(:pageview)])
populate_stats(from_site, [build(:pageview)])
event_before = get_event_by_domain(from_site.domain)
session_before = get_session_by_domain(from_site.domain)
event_before = get_event_by_domain(from_site.domain)
session_before = get_session_by_domain(from_site.domain)
SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
event_after = get_event_by_domain(to_site.domain)
session_after = get_session_by_domain(to_site.domain)
event_after = get_event_by_domain(to_site.domain)
session_after = get_session_by_domain(to_site.domain)
assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}
assert session_before == %ClickhouseSession{session_after | transferred_from: ""}
assert event_after.transferred_from == from_site.domain
assert session_after.transferred_from == from_site.domain
assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}
assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}
assert session_before == %ClickhouseSession{session_after | transferred_from: ""}
assert event_after.transferred_from == from_site.domain
assert session_after.transferred_from == from_site.domain
end
end
test "transfers all events and sessions" do
from_site = insert(:site)
to_site = insert(:site)
if Plausible.v2?() do
:ok
else
from_site = insert(:site)
to_site = insert(:site)
populate_stats(from_site, [
build(:pageview, user_id: 123),
build(:event, name: "Signup", user_id: 123),
build(:pageview, user_id: 456),
build(:event, name: "Signup", user_id: 789)
])
populate_stats(from_site, [
build(:pageview, user_id: 123),
build(:event, name: "Signup", user_id: 123),
build(:pageview, user_id: 456),
build(:event, name: "Signup", user_id: 789)
])
SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
transferred_events =
ClickhouseRepo.all(
from e in Plausible.ClickhouseEvent, where: e.domain == ^to_site.domain
)
transferred_events =
ClickhouseRepo.all(
from e in Plausible.ClickhouseEvent, where: e.domain == ^to_site.domain
)
transferred_sessions =
ClickhouseRepo.all(
from e in Plausible.ClickhouseSession, where: e.domain == ^to_site.domain
)
transferred_sessions =
ClickhouseRepo.all(
from e in Plausible.ClickhouseSession, where: e.domain == ^to_site.domain
)
assert length(transferred_events) == 4
assert length(transferred_sessions) == 3
assert length(transferred_events) == 4
assert length(transferred_sessions) == 3
end
end
test "updates stats_start_date on site record" do
from_site = insert(:site)
to_site = insert(:site)
if Plausible.v2?() do
:ok
else
from_site = insert(:site)
to_site = insert(:site)
populate_stats(from_site, [build(:pageview, timestamp: ~N[2022-01-01 13:21:00])])
populate_stats(from_site, [build(:pageview, timestamp: ~N[2022-01-01 13:21:00])])
SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
assert Repo.reload(to_site).stats_start_date == ~D[2022-01-01]
assert Repo.reload(to_site).stats_start_date == ~D[2022-01-01]
end
end
test "updates native_stats_start_date on based on the from_site record" do
from_site = insert(:site, native_stats_start_at: ~N[2022-01-01 01:00:00])
to_site = insert(:site)
if Plausible.v2?() do
:ok
else
from_site = insert(:site, native_stats_start_at: ~N[2022-01-01 01:00:00])
to_site = insert(:site)
populate_stats(from_site, [build(:pageview, timestamp: ~N[2022-01-01 13:21:00])])
populate_stats(from_site, [build(:pageview, timestamp: ~N[2022-01-01 13:21:00])])
SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
assert Repo.reload(to_site).native_stats_start_at == ~N[2022-01-01 01:00:00]
assert Repo.reload(to_site).native_stats_start_at == ~N[2022-01-01 01:00:00]
end
end
test "session_transfer_query" do

42
test/plausible/site/gate_keeper_test.exs
View File

@ -19,20 +19,21 @@ defmodule Plausible.Site.GateKeeperTest do
test "site from cache with no ingest_rate_limit_threshold is allowed", %{test: test, opts: opts} do
domain = "site1.example.com"
add_site_and_refresh_cache(test, domain: domain)
assert :allow = GateKeeper.check(domain, opts)
%{id: site_id} = add_site_and_refresh_cache(test, domain: domain)
assert {:allow, ^site_id} = GateKeeper.check(domain, opts)
end
test "rate limiting works with threshold", %{test: test, opts: opts} do
domain = "site1.example.com"
add_site_and_refresh_cache(test,
domain: domain,
ingest_rate_limit_threshold: 1,
ingest_rate_limit_scale_seconds: 60
)
%{id: site_id} =
add_site_and_refresh_cache(test,
domain: domain,
ingest_rate_limit_threshold: 1,
ingest_rate_limit_scale_seconds: 60
)
assert :allow = GateKeeper.check(domain, opts)
assert {:allow, ^site_id} = GateKeeper.check(domain, opts)
assert {:deny, :throttle} = GateKeeper.check(domain, opts)
assert {:deny, :throttle} = GateKeeper.check(domain, opts)
end
@ -40,17 +41,18 @@ defmodule Plausible.Site.GateKeeperTest do
test "rate limiting works with scale window", %{test: test, opts: opts} do
domain = "site1.example.com"
add_site_and_refresh_cache(test,
domain: domain,
ingest_rate_limit_threshold: 1,
ingest_rate_limit_scale_seconds: 1
)
%{id: site_id} =
add_site_and_refresh_cache(test,
domain: domain,
ingest_rate_limit_threshold: 1,
ingest_rate_limit_scale_seconds: 1
)
assert :allow = GateKeeper.check(domain, opts)
assert {:allow, ^site_id} = GateKeeper.check(domain, opts)
Process.sleep(1)
assert {:deny, :throttle} = GateKeeper.check(domain, opts)
Process.sleep(1_000)
assert :allow = GateKeeper.check(domain, opts)
assert {:allow, ^site_id} = GateKeeper.check(domain, opts)
end
test "rate limiting prioritises cache lookups", %{test: test, opts: opts} do
@ -67,8 +69,9 @@ defmodule Plausible.Site.GateKeeperTest do
# We need some dummy site, otherwise the cache won't refresh in case the DB
# is completely empty
insert(:site)
deleted_site_id = site.id
assert :allow = GateKeeper.check(domain, opts)
assert {:allow, ^deleted_site_id} = GateKeeper.check(domain, opts)
:ok = Cache.refresh_all(opts[:cache_opts])
assert {:deny, :not_found} = GateKeeper.check(domain, opts)
end
@ -86,18 +89,19 @@ defmodule Plausible.Site.GateKeeperTest do
ingest_rate_limit_scale_seconds: 600
)
assert :allow = GateKeeper.check(domain, opts)
site_id = site.id
assert {:allow, ^site_id} = GateKeeper.check(domain, opts)
assert {:deny, :throttle} = GateKeeper.check(domain, opts)
{:ok, :broken} = break_hammer(site)
log =
capture_log(fn ->
assert :allow = GateKeeper.check(domain, opts)
assert {:allow, ^site_id} = GateKeeper.check(domain, opts)
end)
assert log =~ "Error checking rate limit for 'ingest:site:causingerrors.example.com'"
assert log =~ "Falling back to: allow"
assert log =~ "Falling back to: :allow"
end
# We need a way to force Hammer to error-out on Hammer.check_rate/3.

454
test/plausible_web/controllers/api/external_controller_test.exs
View File

File diff suppressed because it is too large Load Diff

187
test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
View File

@ -97,10 +97,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "aggregates a single metric", %{conn: conn, site: site} do
populate_stats([
build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
populate_stats(site, [
build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -120,12 +120,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
conn: conn,
site: site
} do
populate_stats([
build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview, user_id: 456, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, user_id: 456, domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
populate_stats(site, [
build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview, user_id: 456, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, user_id: 456, timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -145,10 +145,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
conn: conn,
site: site
} do
populate_stats([
build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
populate_stats(site, [
build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -171,19 +171,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
describe "comparisons" do
test "compare period=day with previous period", %{conn: conn, site: site} do
populate_stats([
build(:pageview, domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]),
populate_stats(site, [
build(:pageview, timestamp: ~N[2020-12-31 00:00:00]),
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -204,19 +202,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "compare period=6mo with previous period", %{conn: conn, site: site} do
populate_stats([
build(:pageview, domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]),
populate_stats(site, [
build(:pageview, timestamp: ~N[2020-12-31 00:00:00]),
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-02-01 00:25:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-03-01 00:00:00])
build(:pageview, timestamp: ~N[2021-03-01 00:00:00])
])
conn =
@ -239,19 +235,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
describe "filters" do
test "can filter by source", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
referrer_source: "Google",
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -272,20 +266,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "can filter by no source/referrer", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview,
referrer_source: "Google",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@ -308,19 +299,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "can filter by referrer", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
referrer: "https://facebook.com",
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -341,19 +330,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "can filter by utm_medium", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
utm_medium: "social",
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -374,19 +361,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "can filter by utm_source", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
utm_source: "Twitter",
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -407,19 +392,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "can filter by utm_campaign", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
utm_campaign: "profile",
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -440,19 +423,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "can filter by device type", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
screen_size: "Desktop",
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -473,19 +454,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "can filter by browser", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
browser: "Chrome",
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -506,20 +485,18 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "can filter by browser version", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
browser: "Chrome",
browser_version: "56",
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -540,19 +517,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "can filter by operating system", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
operating_system: "Mac",
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -573,19 +548,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "can filter by operating system version", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
operating_system_version: "10.5",
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -606,19 +579,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "can filter by country", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
country_code: "EE",
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -642,22 +613,19 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
conn: conn,
site: site
} do
populate_stats([
populate_stats(site, [
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
pathname: "/blogpost",
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview,
pathname: "/blogpost",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@ -680,25 +648,22 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "filtering by event:name", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:event,
name: "Signup",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Signup",
domain: site.domain,
user_id: @user_id,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event,
name: "Signup",
domain: site.domain,
user_id: @user_id,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -717,23 +682,20 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
test "combining filters", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
pathname: "/blogpost",
country_code: "EE",
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview,
pathname: "/blogpost",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@ -830,15 +792,54 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
site: site
} do
create_sessions([
%{domain: site.domain, session_id: 1000, country_code: "EE", sign: 1, events: 1},
%{domain: site.domain, session_id: 1000, country_code: "EE", sign: -1, events: 1},
%{domain: site.domain, session_id: 1000, country_code: "EE", sign: 1, events: 2}
%{
domain: site.domain,
site_id: site.id,
session_id: 1000,
country_code: "EE",
sign: 1,
events: 1
},
%{
domain: site.domain,
site_id: site.id,
session_id: 1000,
country_code: "EE",
sign: -1,
events: 1
},
%{
domain: site.domain,
site_id: site.id,
session_id: 1000,
country_code: "EE",
sign: 1,
events: 2
}
])
create_events([
%{domain: site.domain, session_id: 1000, country_code: "EE", name: "pageview"},
%{domain: site.domain, session_id: 1000, country_code: "EE", name: "pageview"},
%{domain: site.domain, session_id: 1000, country_code: "EE", name: "pageview"}
%{
domain: site.domain,
site_id: site.id,
session_id: 1000,
country_code: "EE",
name: "pageview"
},
%{
domain: site.domain,
site_id: site.id,
session_id: 1000,
country_code: "EE",
name: "pageview"
},
%{
domain: site.domain,
site_id: site.id,
session_id: 1000,
country_code: "EE",
name: "pageview"
}
])
conn =

211
test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
View File

@ -131,20 +131,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by visit:source", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
referrer_source: "Google",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
referrer_source: "Google",
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview,
referrer_source: "",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@ -166,10 +163,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by visit:country", %{conn: conn, site: site} do
populate_stats([
build(:pageview, country_code: "EE", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, country_code: "EE", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview, country_code: "US", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
populate_stats(site, [
build(:pageview, country_code: "EE", timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, country_code: "EE", timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview, country_code: "US", timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -189,20 +186,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by visit:referrer", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
referrer: "https://ref.com",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
referrer: "https://ref.com",
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview,
referrer: "",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@ -224,20 +218,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by visit:utm_medium", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
utm_medium: "Search",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
utm_medium: "Search",
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview,
utm_medium: "",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@ -259,20 +250,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by visit:utm_source", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
utm_source: "Google",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
utm_source: "Google",
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview,
utm_source: "",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@ -390,20 +378,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by visit:device", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
screen_size: "Desktop",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
screen_size: "Desktop",
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview,
screen_size: "Mobile",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@ -425,20 +410,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by visit:os", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
operating_system: "Mac",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
operating_system: "Mac",
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview,
operating_system: "Windows",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@ -460,20 +442,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by visit:os_version", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
operating_system_version: "10.5",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
operating_system_version: "10.5",
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview,
operating_system_version: "10.6",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@ -495,10 +474,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by visit:browser", %{conn: conn, site: site} do
populate_stats([
build(:pageview, browser: "Firefox", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, browser: "Firefox", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview, browser: "Safari", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
populate_stats(site, [
build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview, browser: "Safari", timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -518,20 +497,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by visit:browser_version", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
browser_version: "56",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
browser_version: "56",
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview,
browser_version: "57",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@ -553,12 +529,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by event:page", %{conn: conn, site: site} do
populate_stats([
build(:pageview, pathname: "/", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, pathname: "/", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
populate_stats(site, [
build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview,
pathname: "/plausible.io",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@ -583,12 +558,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
conn: conn,
site: site
} do
populate_stats([
build(:pageview, pathname: "/", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, pathname: "/", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
populate_stats(site, [
build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview,
pathname: "/plausible.io",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@ -609,19 +583,16 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
describe "custom events" do
test "can breakdown by event:name", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:event,
name: "Signup",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Signup",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
)
])
@ -643,40 +614,34 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "can breakdown by event:name with visitors and events metrics", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
domain: site.domain,
pathname: "/non-existing",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "404",
domain: site.domain,
pathname: "/non-existing",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
domain: site.domain,
pathname: "/non-existing",
timestamp: ~N[2021-01-01 00:00:01]
),
build(:pageview,
domain: site.domain,
pathname: "/non-existing",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:00:02]
),
build(:event,
name: "404",
domain: site.domain,
pathname: "/non-existing",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:00:02]
),
build(:pageview,
domain: site.domain,
pathname: "/non-existing",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:00:03]
@ -701,39 +666,34 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "can breakdown by event:name while filtering for something", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:event,
name: "Signup",
pathname: "/pageA",
browser: "Chrome",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Signup",
pathname: "/pageA",
browser: "Chrome",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Signup",
pathname: "/pageA",
browser: "Safari",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Signup",
pathname: "/pageB",
browser: "Chrome",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/pageA",
browser: "Chrome",
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
)
])
@ -759,21 +719,18 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
conn: conn,
site: site
} do
populate_stats([
populate_stats(site, [
build(:pageview,
referrer_source: "Google",
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Signup",
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
domain: site.domain,
referrer_source: "Twitter",
timestamp: ~N[2021-01-01 00:25:00]
)
@ -796,26 +753,22 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "can breakdown by event:name when filtering by event:page", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
pathname: "/pageA",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/pageA",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
pathname: "/pageA",
name: "Signup",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/pageB",
domain: site.domain,
referrer_source: "Twitter",
timestamp: ~N[2021-01-01 00:25:00]
)
@ -839,28 +792,24 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "can breakdown by event:page when filtering by event:name", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:event,
name: "Signup",
pathname: "/pageA",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Signup",
pathname: "/pageA",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Signup",
pathname: "/pageB",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/pageB",
domain: site.domain,
referrer_source: "Twitter",
timestamp: ~N[2021-01-01 00:25:00]
)
@ -911,33 +860,29 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by custom event property", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:event,
name: "Purchase",
"meta.key": ["package"],
"meta.value": ["business"],
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Purchase",
"meta.key": ["package"],
"meta.value": ["personal"],
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Purchase",
"meta.key": ["package"],
"meta.value": ["business"],
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event,
name: "Some other event",
"meta.key": ["package"],
"meta.value": ["business"],
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
)
])
@ -960,45 +905,39 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by custom event property, with (none)", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["16"],
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["16"],
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["16"],
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Purchase",
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["14"],
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["14"],
domain: site.domain,
timestamp: ~N[2021-01-01 00:26:00]
)
])
@ -1022,33 +961,29 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by custom event property, limited", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["16"],
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["18"],
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["14"],
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["14"],
domain: site.domain,
timestamp: ~N[2021-01-01 00:26:00]
)
])
@ -1072,40 +1007,35 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "breakdown by custom event property, paginated", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["16"],
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["16"],
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["18"],
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["14"],
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event,
name: "Purchase",
"meta.key": ["cost"],
"meta.value": ["14"],
domain: site.domain,
timestamp: ~N[2021-01-01 00:26:00]
)
])
@ -1135,14 +1065,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
insert(:goal, %{domain: site.domain, event_name: "Purchase"})
insert(:goal, %{domain: site.domain, page_path: "/test"})
populate_stats([
populate_stats(site, [
build(:pageview,
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00],
pathname: "/test"
),
build(:pageview,
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:01],
pathname: "/test",
"meta.key": ["method"],
@ -1150,14 +1078,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
),
build(:event,
name: "404",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:02],
"meta.key": ["method"],
"meta.value": ["HTTP"]
),
build(:event,
name: "Purchase",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:02],
"meta.key": ["method"],
"meta.value": ["HTTPS"]
@ -1165,14 +1091,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
build(:event,
name: "404",
timestamp: ~N[2021-01-01 00:00:03],
domain: site.domain,
"meta.key": ["OS", "method"],
"meta.value": ["Linux", "HTTP"]
),
build(:event,
name: "404",
timestamp: ~N[2021-01-01 00:00:04],
domain: site.domain,
"meta.key": ["version"],
"meta.value": ["1"]
)
@ -1213,29 +1137,25 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
describe "filtering" do
test "event:page filter for breakdown by session props", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
pathname: "/ignore",
browser: "Chrome",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/plausible.io",
browser: "Chrome",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/plausible.io",
browser: "Chrome",
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:pageview,
browser: "Safari",
pathname: "/plausible.io",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@ -1409,25 +1329,21 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "IN filter for event:page", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
pathname: "/ignore",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/plausible.io",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/plausible.io",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/important-page",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@ -1450,29 +1366,25 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "IN filter for visit:browser", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
pathname: "/ignore",
browser: "Firefox",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/plausible.io",
browser: "Chrome",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/plausible.io",
browser: "Safari",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/important-page",
browser: "Safari",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@ -1495,25 +1407,21 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "IN filter for visit:entry_page", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
pathname: "/ignore",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/plausible.io",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/plausible.io",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/important-page",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@ -1537,25 +1445,21 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "IN filter for event:name", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:event,
name: "Signup",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Signup",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Login",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Irrelevant",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@ -1603,10 +1507,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
describe "pagination" do
test "can limit results", %{conn: conn, site: site} do
populate_stats([
build(:pageview, pathname: "/a", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, pathname: "/b", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview, pathname: "/c", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
populate_stats(site, [
build(:pageview, pathname: "/a", timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, pathname: "/b", timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview, pathname: "/c", timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -1623,11 +1527,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "does not repeat results", %{conn: conn, site: site} do
populate_stats([
build(:pageview, %{domain: site.domain, "meta.key": ["item"], "meta.value": ["apple"]}),
build(:pageview, %{domain: site.domain, "meta.key": ["item"], "meta.value": ["kiwi"]}),
build(:pageview, %{domain: site.domain, "meta.key": ["item"], "meta.value": ["pineapple"]}),
build(:pageview, %{domain: site.domain, "meta.key": ["item"], "meta.value": ["grapes"]})
populate_stats(site, [
build(:pageview, %{"meta.key": ["item"], "meta.value": ["apple"]}),
build(:pageview, %{"meta.key": ["item"], "meta.value": ["kiwi"]}),
build(:pageview, %{"meta.key": ["item"], "meta.value": ["pineapple"]}),
build(:pageview, %{"meta.key": ["item"], "meta.value": ["grapes"]})
])
params = %{
@ -1693,10 +1597,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "can paginate results", %{conn: conn, site: site} do
populate_stats([
build(:pageview, pathname: "/a", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, pathname: "/b", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview, pathname: "/c", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
populate_stats(site, [
build(:pageview, pathname: "/a", timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, pathname: "/b", timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview, pathname: "/c", timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -1820,13 +1724,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "filter by custom event property", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:event,
name: "Purchase",
"meta.key": ["package"],
"meta.value": ["business"],
browser: "Chrome",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
@ -1834,7 +1737,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
"meta.key": ["package"],
"meta.value": ["business"],
browser: "Safari",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
@ -1842,7 +1744,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
"meta.key": ["package"],
"meta.value": ["business"],
browser: "Safari",
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event,
@ -1850,7 +1751,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
"meta.key": ["package"],
"meta.value": ["personal"],
browser: "IE",
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
)
])
@ -1873,23 +1773,20 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
end
test "all metrics for breakdown by event prop", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
user_id: 1,
pathname: "/",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: 1,
pathname: "/plausible.io",
domain: site.domain,
timestamp: ~N[2021-01-01 00:10:00]
),
build(:pageview, pathname: "/", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),
build(:pageview,
pathname: "/plausible.io",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])

143
test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs
View File

@ -261,9 +261,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "shows last 7 days of visitors", %{conn: conn, site: site} do
populate_stats([
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-07 23:59:00])
populate_stats(site, [
build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, timestamp: ~N[2021-01-07 23:59:00])
])
conn =
@ -287,10 +287,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "shows last 6 months of visitors", %{conn: conn, site: site} do
populate_stats([
build(:pageview, domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
populate_stats(site, [
build(:pageview, timestamp: ~N[2020-12-31 00:00:00]),
build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -313,11 +313,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "shows last 12 months of visitors", %{conn: conn, site: site} do
populate_stats([
build(:pageview, domain: site.domain, timestamp: ~N[2020-02-01 00:00:00]),
build(:pageview, domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
populate_stats(site, [
build(:pageview, timestamp: ~N[2020-02-01 00:00:00]),
build(:pageview, timestamp: ~N[2020-12-31 00:00:00]),
build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -346,11 +346,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "shows last 12 months of visitors with interval daily", %{conn: conn, site: site} do
populate_stats([
build(:pageview, domain: site.domain, timestamp: ~N[2020-02-01 00:00:00]),
build(:pageview, domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
populate_stats(site, [
build(:pageview, timestamp: ~N[2020-02-01 00:00:00]),
build(:pageview, timestamp: ~N[2020-12-31 00:00:00]),
build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -365,10 +365,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "shows a custom range with daily interval", %{conn: conn, site: site} do
populate_stats([
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-02 00:00:00])
populate_stats(site, [
build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, timestamp: ~N[2021-01-02 00:00:00])
])
conn =
@ -387,11 +387,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "shows a custom range with monthly interval", %{conn: conn, site: site} do
populate_stats([
build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2020-12-01 00:00:00]),
build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2020-12-01 00:05:00]),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-02 00:00:00])
populate_stats(site, [
build(:pageview, user_id: @user_id, timestamp: ~N[2020-12-01 00:00:00]),
build(:pageview, user_id: @user_id, timestamp: ~N[2020-12-01 00:05:00]),
build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, timestamp: ~N[2021-01-02 00:00:00])
])
conn =
@ -425,13 +425,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
describe "filters" do
test "can filter by source", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
referrer_source: "Google",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -447,11 +446,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "can filter by no source/referrer", %{conn: conn, site: site} do
populate_stats([
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
populate_stats(site, [
build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview,
referrer_source: "Google",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@ -469,13 +467,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "can filter by referrer", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
referrer: "https://facebook.com",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -491,13 +488,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "can filter by utm_medium", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
utm_medium: "social",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -513,13 +509,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "can filter by utm_source", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
utm_source: "Twitter",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -535,13 +530,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "can filter by utm_campaign", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
utm_campaign: "profile",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -557,13 +551,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "can filter by device type", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
screen_size: "Desktop",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -579,20 +572,18 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "can filter by browser", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
browser: "Chrome",
browser_version: "56.1",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
browser: "Chrome",
browser_version: "55",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -608,26 +599,23 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "can filter by operating system", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
operating_system: "Mac",
operating_system_version: "10.5",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
operating_system: "Something else",
operating_system_version: "10.5",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
operating_system: "Mac",
operating_system_version: "10.4",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -643,22 +631,20 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "can filter by country", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
user_id: @user_id,
country_code: "EE",
operating_system_version: "10.5",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
country_code: "EE",
operating_system_version: "10.5",
domain: site.domain,
timestamp: ~N[2021-01-01 00:15:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -685,26 +671,22 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
conn: conn,
site: site
} do
populate_stats([
populate_stats(site, [
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/hello",
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:05:00]
),
build(:pageview,
pathname: "/hello",
domain: site.domain,
timestamp: ~N[2021-01-01 05:00:00]
),
build(:pageview,
pathname: "/goobye",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])
@ -732,13 +714,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "can filter by event:name", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:event,
name: "Signup",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
])
conn =
@ -754,33 +735,29 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
end
test "filter by custom event property", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:event,
name: "Purchase",
"meta.key": ["package"],
"meta.value": ["business"],
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Purchase",
"meta.key": ["package"],
"meta.value": ["business"],
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Purchase",
"meta.key": ["package"],
"meta.value": ["personal"],
domain: site.domain,
timestamp: ~N[2021-01-01 00:25:00]
),
build(:event,
name: "Purchase",
"meta.key": ["package"],
"meta.value": ["business"],
domain: site.domain,
timestamp: ~N[2021-01-02 00:25:00]
)
])
@ -801,19 +778,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
describe "metrics" do
test "shows pageviews,visits,views_per_visit for last 7d", %{conn: conn, site: site} do
populate_stats([
populate_stats(site, [
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:05:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-07 23:59:00])
build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, timestamp: ~N[2021-01-07 23:59:00])
])
conn =
@ -876,30 +851,26 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
conn: conn,
site: site
} do
populate_stats([
populate_stats(site, [
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-01 00:05:00]
),
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-03 00:00:00]
),
build(:pageview,
user_id: @user_id,
domain: site.domain,
timestamp: ~N[2021-01-03 00:01:00]
),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-03 00:00:00]),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-03 00:00:00]),
build(:pageview, domain: site.domain, timestamp: ~N[2021-01-07 23:59:00])
build(:pageview, timestamp: ~N[2021-01-03 00:00:00]),
build(:pageview, timestamp: ~N[2021-01-03 00:00:00]),
build(:pageview, timestamp: ~N[2021-01-07 23:59:00])
])
conn =

2
test/plausible_web/controllers/api/internal_controller_test.exs
View File

@ -14,7 +14,7 @@ defmodule PlausibleWeb.Api.InternalControllerTest do
test "is READY when site has at least 1 pageview", %{conn: conn, user: user} do
site = insert(:site, members: [user])
Plausible.TestUtils.create_pageviews([%{domain: site.domain}])
Plausible.TestUtils.create_pageviews([%{site: site}])
conn = get(conn, "/api/#{site.domain}/status")

35
test/plausible_web/controllers/api/stats_controller/conversions_test.exs
View File

@ -653,6 +653,41 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
insert(:goal, %{domain: site.domain, page_path: "/*"})
insert(:goal, %{domain: site.domain, page_path: "/**"})
populate_stats(site, [
build(:pageview,
pathname: "/hum",
timestamp: ~N[2019-07-01 23:00:00]
),
build(:pageview,
pathname: "/register",
timestamp: ~N[2019-07-01 23:00:00]
),
build(:pageview,
pathname: "/reg",
timestamp: ~N[2019-07-01 23:00:00]
),
build(:pageview,
pathname: "/billing/success",
timestamp: ~N[2019-07-01 23:00:00]
),
build(:pageview,
pathname: "/billing/upgrade/success",
timestamp: ~N[2019-07-01 23:00:00]
),
build(:pageview,
pathname: "/signup/new",
timestamp: ~N[2019-07-01 23:00:00]
),
build(:pageview,
pathname: "/signup/new/2",
timestamp: ~N[2019-07-01 23:00:00]
),
build(:pageview,
pathname: "/signup/new/3",
timestamp: ~N[2019-07-01 23:00:00]
)
])
conn =
get(
conn,

5
test/plausible_web/controllers/api/stats_controller/current_visitors_test.exs
View File

@ -5,6 +5,11 @@ defmodule PlausibleWeb.Api.StatsController.CurrentVisitorsTest do
setup [:create_user, :log_in, :create_site]
test "returns unique users in the last 5 minutes", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, user_id: 123),
build(:pageview, user_id: 456)
])
conn = get(conn, "/api/stats/#{site.domain}/current-visitors")
assert json_response(conn, 200) == 2

71
test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
View File

@ -5,23 +5,37 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
setup [:create_user, :log_in, :create_site]
test "returns suggestions for pages without a query", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/"),
build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/register"),
build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/contact"),
build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/irrelevant")
])
conn = get(conn, "/api/stats/#{site.domain}/suggestions/page?period=month&date=2019-01-01")
assert json_response(conn, 200) == [
%{"label" => "/", "value" => "/"},
%{"label" => "/register", "value" => "/register"},
%{"label" => "/contact", "value" => "/contact"},
%{"label" => "/irrelevant", "value" => "/irrelevant"}
%{"label" => "/irrelevant", "value" => "/irrelevant"},
%{"label" => "/register", "value" => "/register"}
]
end
test "returns suggestions for pages with a query", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/"),
build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/register"),
build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/contact"),
build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/irrelevant")
])
conn =
get(conn, "/api/stats/#{site.domain}/suggestions/page?period=month&date=2019-01-01&q=re")
assert json_response(conn, 200) == [
%{"label" => "/register", "value" => "/register"},
%{"label" => "/irrelevant", "value" => "/irrelevant"}
%{"label" => "/irrelevant", "value" => "/irrelevant"},
%{"label" => "/register", "value" => "/register"}
]
end
@ -42,16 +56,25 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
end
test "returns suggestions for sources", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, timestamp: ~N[2019-01-01 23:00:00], referrer_source: "Bing"),
build(:pageview, timestamp: ~N[2019-01-01 23:00:00], referrer_source: "10words")
])
conn =
get(conn, "/api/stats/#{site.domain}/suggestions/source?period=month&date=2019-01-01")
assert json_response(conn, 200) == [
%{"label" => "10words", "value" => "10words"},
%{"label" => "Bing", "value" => "Bing"}
%{"label" => "Bing", "value" => "Bing"},
%{"label" => "10words", "value" => "10words"}
]
end
test "returns suggestions for countries", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/", country_code: "US")
])
conn =
get(
conn,
@ -106,6 +129,10 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
end
test "returns suggestions for screen sizes", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/", screen_size: "Desktop")
])
conn =
get(conn, "/api/stats/#{site.domain}/suggestions/screen?period=month&date=2019-01-01")
@ -113,6 +140,10 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
end
test "returns suggestions for browsers", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/", browser: "Chrome")
])
conn =
get(conn, "/api/stats/#{site.domain}/suggestions/browser?period=month&date=2019-01-01")
@ -122,6 +153,14 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
test "returns suggestions for browser versions", %{conn: conn, site: site} do
filters = Jason.encode!(%{browser: "Chrome"})
populate_stats(site, [
build(:pageview,
timestamp: ~N[2019-01-01 00:00:00],
browser: "Chrome",
browser_version: "78.0"
)
])
conn =
get(
conn,
@ -132,6 +171,10 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
end
test "returns suggestions for OS", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, timestamp: ~N[2019-01-01 00:00:00], operating_system: "Mac")
])
conn = get(conn, "/api/stats/#{site.domain}/suggestions/os?period=month&date=2019-01-01")
assert json_response(conn, 200) == [%{"value" => "Mac", "label" => "Mac"}]
@ -140,6 +183,14 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
test "returns suggestions for OS versions", %{conn: conn, site: site} do
filters = Jason.encode!(%{os: "Mac"})
populate_stats(site, [
build(:pageview,
timestamp: ~N[2019-01-01 00:00:00],
operating_system: "Mac",
operating_system_version: "10.15"
)
])
conn =
get(
conn,
@ -162,6 +213,14 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
end
test "returns suggestions for referrers", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
timestamp: ~N[2019-01-01 23:00:00],
pathname: "/",
referrer: "10words.com/page1"
)
])
conn =
get(conn, "/api/stats/#{site.domain}/suggestions/referrer?period=month&date=2019-01-01")

6
test/plausible_web/controllers/site_controller_test.exs
View File

@ -41,10 +41,12 @@ defmodule PlausibleWeb.SiteControllerTest do
end
test "lists all of your sites with last 24h visitors", %{conn: conn, user: user} do
insert(:site, members: [user], domain: "test-site.com")
site = insert(:site, members: [user])
populate_stats(site, [build(:pageview), build(:pageview), build(:pageview)])
conn = get(conn, "/sites")
assert html_response(conn, 200) =~ "test-site.com"
assert html_response(conn, 200) =~ site.domain
assert html_response(conn, 200) =~ "<b>3</b> visitors in last 24h"
end

6
test/plausible_web/controllers/stats_controller_test.exs
View File

@ -4,9 +4,10 @@ defmodule PlausibleWeb.StatsControllerTest do
describe "GET /:website - anonymous user" do
test "public site - shows site stats", %{conn: conn} do
insert(:site, domain: "public-site.io", public: true)
site = insert(:site, public: true)
populate_stats(site, [build(:pageview)])
conn = get(conn, "/public-site.io")
conn = get(conn, "/#{site.domain}")
assert html_response(conn, 200) =~ "stats-react-container"
end
@ -27,6 +28,7 @@ defmodule PlausibleWeb.StatsControllerTest do
setup [:create_user, :log_in, :create_site]
test "can view stats of a website I've created", %{conn: conn, site: site} do
populate_stats(site, [build(:pageview)])
conn = get(conn, "/" <> site.domain)
assert html_response(conn, 200) =~ "stats-react-container"
end

337
test/support/clickhouse_setup.ex
View File

@ -1,337 +0,0 @@
defmodule Plausible.Test.ClickhouseSetup do
@conversion_1_session_id 123
@conversion_2_session_id 234
def run() do
Plausible.TestUtils.create_events([
%{
name: "pageview",
domain: "test-site.com",
pathname: "/",
country_code: "EE",
browser: "Chrome",
browser_version: "78.0",
operating_system: "Mac",
operating_system_version: "10.15",
screen_size: "Desktop",
referrer_source: "10words",
referrer: "10words.com/page1",
timestamp: ~N[2019-01-01 00:00:00],
session_id: @conversion_1_session_id
},
%{
name: "pageview",
domain: "test-site.com",
pathname: "/",
country_code: "EE",
browser: "Chrome",
browser_version: "78.0",
operating_system: "Mac",
operating_system_version: "10.15",
screen_size: "Desktop",
referrer_source: "10words",
referrer: "10words.com/page2",
timestamp: ~N[2019-01-01 00:00:00]
},
%{
name: "pageview",
domain: "test-site.com",
pathname: "/",
country_code: "US",
browser: "Chrome",
browser_version: "79.0",
operating_system: "Mac",
operating_system_version: "10.16",
screen_size: "Laptop",
referrer_source: "",
referrer: "",
timestamp: ~N[2019-01-01 00:00:00]
},
%{
name: "pageview",
domain: "test-site.com",
pathname: "/contact",
country_code: "GB",
browser: "Firefox",
operating_system: "Android",
screen_size: "Mobile",
referrer_source: "Bing",
timestamp: ~N[2019-01-01 00:00:00]
},
%{name: "pageview", domain: "test-site.com", timestamp: ~N[2019-01-31 00:00:00]},
%{
name: "Signup",
domain: "test-site.com",
session_id: @conversion_1_session_id,
timestamp: ~N[2019-01-01 01:00:00],
"meta.key": ["variant"],
"meta.value": ["A"]
},
%{
name: "Signup",
domain: "test-site.com",
session_id: @conversion_1_session_id,
timestamp: ~N[2019-01-01 02:00:00],
"meta.key": ["variant"],
"meta.value": ["B"]
},
%{
name: "Signup",
domain: "test-site.com",
session_id: @conversion_2_session_id,
timestamp: ~N[2019-01-01 02:00:00],
"meta.key": ["variant"],
"meta.value": ["B"]
},
%{
name: "pageview",
pathname: "/register",
domain: "test-site.com",
session_id: @conversion_1_session_id,
timestamp: ~N[2019-01-01 23:00:00]
},
%{
name: "pageview",
pathname: "/register",
domain: "test-site.com",
session_id: @conversion_2_session_id,
timestamp: ~N[2019-01-01 23:00:00]
},
%{
name: "pageview",
pathname: "/irrelevant",
domain: "test-site.com",
session_id: @conversion_1_session_id,
timestamp: ~N[2019-01-01 23:00:01]
},
%{
name: "pageview",
domain: "test-site.com",
referrer_source: "Google",
timestamp: ~N[2019-02-01 01:00:00]
},
%{
name: "pageview",
domain: "test-site.com",
referrer_source: "Google",
timestamp: ~N[2019-02-01 02:00:00]
},
%{
name: "pageview",
domain: "test-site.com",
referrer: "t.co/some-link",
referrer_source: "Twitter",
timestamp: ~N[2019-03-01 01:00:00]
},
%{
name: "pageview",
domain: "test-site.com",
referrer: "t.co/some-link",
referrer_source: "Twitter",
timestamp: ~N[2019-03-01 01:00:00]
},
%{
name: "pageview",
domain: "test-site.com",
referrer: "t.co/nonexistent-link",
referrer_source: "Twitter",
timestamp: ~N[2019-03-01 02:00:00]
},
%{name: "pageview", domain: "test-site.com"},
%{
name: "pageview",
domain: "test-site.com",
timestamp: Timex.now() |> Timex.shift(minutes: -3)
},
%{
name: "pageview",
domain: "test-site.com",
timestamp: Timex.now() |> Timex.shift(minutes: -6)
},
%{name: "pageview", domain: "tz-test.com", timestamp: ~N[2019-01-01 00:00:00]},
%{name: "pageview", domain: "public-site.io"},
%{
name: "pageview",
domain: "fetch-tweets-test.com",
referrer: "t.co/a-link",
referrer_source: "Twitter"
},
%{
name: "pageview",
domain: "fetch-tweets-test.com",
referrer: "t.co/b-link",
referrer_source: "Twitter",
timestamp: Timex.now() |> Timex.shift(days: -5)
},
%{
name: "pageview",
pathname: "/register",
domain: "test-site.com",
timestamp: ~N[2019-07-01 23:00:00]
},
%{
name: "pageview",
pathname: "/signup/new",
domain: "test-site.com",
timestamp: ~N[2019-07-01 23:00:00]
},
%{
name: "pageview",
pathname: "/billing/success",
domain: "test-site.com",
timestamp: ~N[2019-07-01 23:00:00]
},
%{
name: "pageview",
pathname: "/",
domain: "test-site.com",
timestamp: ~N[2019-07-01 23:00:00]
},
%{
name: "pageview",
pathname: "/reg",
domain: "test-site.com",
timestamp: ~N[2019-07-01 23:00:00]
},
%{
name: "pageview",
pathname: "/signup/new/2",
domain: "test-site.com",
timestamp: ~N[2019-07-01 23:00:00]
},
%{
name: "pageview",
pathname: "/signup/new/3",
domain: "test-site.com",
timestamp: ~N[2019-07-01 23:00:00]
},
%{
name: "pageview",
pathname: "/billing/upgrade/success",
domain: "test-site.com",
timestamp: ~N[2019-07-01 23:00:00]
}
])
Plausible.TestUtils.create_sessions([
%{
domain: "test-site.com",
entry_page: "/",
exit_page: "/",
referrer_source: "10words",
referrer: "10words.com/page1",
utm_medium: "listing",
country_code: "US",
screen_size: "Desktop",
browser: "Chrome",
browser_version: "78.0",
operating_system: "Mac",
operating_system_version: "10.15",
session_id: @conversion_1_session_id,
is_bounce: true,
duration: 100,
start: ~N[2019-01-01 02:00:00],
timestamp: ~N[2019-01-01 02:00:00]
},
%{
domain: "test-site.com",
entry_page: "/",
exit_page: "/",
referrer_source: "10words",
referrer: "10words.com/page1",
utm_medium: "listing",
session_id: @conversion_2_session_id,
is_bounce: false,
duration: 0,
start: ~N[2019-01-01 02:00:00],
timestamp: ~N[2019-01-01 02:00:00]
},
%{
domain: "test-site.com",
entry_page: "/",
exit_page: "/",
referrer_source: "Bing",
referrer: "",
utm_medium: "search",
is_bounce: false,
duration: 100,
start: ~N[2019-01-01 03:00:00],
timestamp: ~N[2019-01-01 03:00:00]
},
%{
domain: "test-site.com",
entry_page: "/",
exit_page: "/",
referrer_source: "Google",
referrer: "",
is_bounce: false,
start: ~N[2019-02-01 01:00:00],
timestamp: ~N[2019-02-01 01:00:00]
},
%{
domain: "test-site.com",
entry_page: "/",
exit_page: "/",
referrer_source: "Google",
referrer: "",
is_bounce: false,
start: ~N[2019-02-01 02:00:00],
timestamp: ~N[2019-02-01 02:00:00]
},
%{
domain: "test-site.com",
entry_page: "/",
exit_page: "/",
referrer: "t.co/some-link",
referrer_source: "Twitter",
start: ~N[2019-03-01 01:00:00],
timestamp: ~N[2019-03-01 01:00:00]
},
%{
domain: "test-site.com",
entry_page: "/",
exit_page: "/",
referrer: "t.co/some-link",
referrer_source: "Twitter",
start: ~N[2019-03-01 01:00:00],
timestamp: ~N[2019-03-01 01:00:00]
},
%{
domain: "test-site.com",
entry_page: "/",
exit_page: "/",
referrer: "t.co/nonexistent-link",
referrer_source: "Twitter",
start: ~N[2019-03-01 02:00:00],
timestamp: ~N[2019-03-01 02:00:00]
},
%{
domain: "test-site.com",
entry_page: "/",
exit_page: "/",
referrer_source: "Bing",
referrer: "bing.com",
start: Timex.now() |> Timex.shift(minutes: -1),
timestamp: Timex.now() |> Timex.shift(minutes: -1)
},
%{
domain: "test-site.com",
entry_page: "/",
exit_page: "/exit",
referrer_source: "10words",
referrer: "10words.com",
start: Timex.now() |> Timex.shift(minutes: -2),
timestamp: Timex.now() |> Timex.shift(minutes: -2)
},
%{
domain: "test-site.com",
entry_page: "/",
exit_page: "/exit",
referrer_source: "10words",
referrer: "10words.com",
start: Timex.now() |> Timex.shift(minutes: -3),
timestamp: Timex.now() |> Timex.shift(minutes: -3)
}
])
end
end

69
test/support/factory.ex
View File

@ -38,19 +38,35 @@ defmodule Plausible.Factory do
def ch_session_factory do
hostname = sequence(:domain, &"example-#{&1}.com")
%Plausible.ClickhouseSession{
sign: 1,
session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
hostname: hostname,
domain: hostname,
entry_page: "/",
pageviews: 1,
events: 1,
start: Timex.now(),
timestamp: Timex.now(),
is_bounce: false
}
if Plausible.v2?() do
%Plausible.ClickhouseSessionV2{
sign: 1,
session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
hostname: hostname,
site_id: Enum.random(1000..10_000),
entry_page: "/",
pageviews: 1,
events: 1,
start: Timex.now(),
timestamp: Timex.now(),
is_bounce: false
}
else
%Plausible.ClickhouseSession{
sign: 1,
session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
hostname: hostname,
domain: hostname,
entry_page: "/",
pageviews: 1,
events: 1,
start: Timex.now(),
timestamp: Timex.now(),
is_bounce: false
}
end
end
def pageview_factory do
@ -65,14 +81,25 @@ defmodule Plausible.Factory do
def event_factory do
hostname = sequence(:domain, &"example-#{&1}.com")
%Plausible.ClickhouseEvent{
hostname: hostname,
domain: hostname,
pathname: "/",
timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second),
user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate())
}
if Plausible.v2?() do
%Plausible.ClickhouseEventV2{
hostname: hostname,
site_id: Enum.random(1000..10_000),
pathname: "/",
timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second),
user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate())
}
else
%Plausible.ClickhouseEvent{
hostname: hostname,
domain: hostname,
pathname: "/",
timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second),
user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate())
}
end
end
def goal_factory do

65
test/support/test_utils.ex
View File

@ -37,7 +37,6 @@ defmodule Plausible.TestUtils do
def create_site(%{user: user}) do
site =
Factory.insert(:site,
domain: "test-site.com",
members: [user]
)
@ -73,30 +72,63 @@ defmodule Plausible.TestUtils do
def create_pageviews(pageviews) do
pageviews =
Enum.map(pageviews, fn pageview ->
pageview =
if Plausible.v2?() do
pageview
|> Map.delete(:site)
|> Map.put(:site_id, pageview.site.id)
else
pageview
|> Map.delete(:site)
|> Map.put(:domain, pageview.site.domain)
end
Factory.build(:pageview, pageview)
|> Map.from_struct()
|> Map.delete(:__meta__)
|> update_in([:timestamp], &to_naive_truncate/1)
end)
Plausible.IngestRepo.insert_all(Plausible.ClickhouseEvent, pageviews)
if Plausible.v2?() do
Plausible.IngestRepo.insert_all(Plausible.ClickhouseEventV2, pageviews)
else
Plausible.IngestRepo.insert_all(Plausible.ClickhouseEvent, pageviews)
end
end
def create_events(events) do
events =
Enum.map(events, fn event ->
event =
if Plausible.v2?() do
Map.delete(event, :domain)
else
Map.delete(event, :site_id)
end
Factory.build(:event, event)
|> Map.from_struct()
|> Map.delete(:__meta__)
|> update_in([:timestamp], &to_naive_truncate/1)
end)
Plausible.IngestRepo.insert_all(Plausible.ClickhouseEvent, events)
if Plausible.v2?() do
Plausible.IngestRepo.insert_all(Plausible.ClickhouseEventV2, events)
else
Plausible.IngestRepo.insert_all(Plausible.ClickhouseEvent, events)
end
end
def create_sessions(sessions) do
sessions =
Enum.map(sessions, fn session ->
session =
if Plausible.v2?() do
Map.delete(session, :domain)
else
Map.delete(session, :site_id)
end
Factory.build(:ch_session, session)
|> Map.from_struct()
|> Map.delete(:__meta__)
@ -104,7 +136,11 @@ defmodule Plausible.TestUtils do
|> update_in([:start], &to_naive_truncate/1)
end)
Plausible.IngestRepo.insert_all(Plausible.ClickhouseSession, sessions)
if Plausible.v2?() do
Plausible.IngestRepo.insert_all(Plausible.ClickhouseSessionV2, sessions)
else
Plausible.IngestRepo.insert_all(Plausible.ClickhouseSession, sessions)
end
end
def log_in(%{user: user, conn: conn}) do
@ -134,6 +170,9 @@ defmodule Plausible.TestUtils do
def populate_stats(site, events) do
Enum.map(events, fn event ->
case event do
%Plausible.ClickhouseEventV2{} ->
Map.put(event, :site_id, site.id)
%Plausible.ClickhouseEvent{} ->
Map.put(event, :domain, site.domain)
@ -158,6 +197,9 @@ defmodule Plausible.TestUtils do
end)
|> Enum.split_with(fn event ->
case event do
%Plausible.ClickhouseEventV2{} ->
true
%Plausible.ClickhouseEvent{} ->
true
@ -174,11 +216,22 @@ defmodule Plausible.TestUtils do
sessions =
Enum.reduce(events, %{}, fn event, sessions ->
session_id = Plausible.Session.CacheStore.on_event(event, nil)
Map.put(sessions, {event.domain, event.user_id}, session_id)
if Plausible.v2?() do
Map.put(sessions, {event.site_id, event.user_id}, session_id)
else
Map.put(sessions, {event.domain, event.user_id}, session_id)
end
end)
Enum.each(events, fn event ->
event = Map.put(event, :session_id, sessions[{event.domain, event.user_id}])
event =
if Plausible.v2?() do
Map.put(event, :session_id, sessions[{event.site_id, event.user_id}])
else
Map.put(event, :session_id, sessions[{event.domain, event.user_id}])
end
Plausible.Event.WriteBuffer.insert(event)
end)

9
test/test_helper.exs
View File

@ -1,7 +1,14 @@
{:ok, _} = Application.ensure_all_started(:ex_machina)
Plausible.Test.ClickhouseSetup.run()
Mox.defmock(Plausible.HTTPClient.Mock, for: Plausible.HTTPClient.Interface)
FunWithFlags.enable(:visits_metric)
ExUnit.start(exclude: :slow)
Application.ensure_all_started(:double)
Ecto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual)
if Plausible.v2?() do
IO.puts("Running tests against v2 schema")
else
IO.puts(
"Running tests against v1 schema. Use: `V2_MIGRATION_DONE=1 mix test` for secondary run."
)
end

5
test/workers/send_check_stats_emails_test.exs
View File

@ -25,6 +25,8 @@ defmodule Plausible.Workers.SendCheckStatsEmailsTest do
test "does not send an email if the user has configured a weekly report" do
user = insert(:user, inserted_at: days_ago(9), last_seen: days_ago(7))
site = insert(:site, domain: "test-site.com", members: [user])
populate_stats(site, [build(:pageview)])
insert(:weekly_report, site: site, recipients: ["user@email.com"])
perform_job(SendCheckStatsEmails, %{})
@ -34,7 +36,8 @@ defmodule Plausible.Workers.SendCheckStatsEmailsTest do
test "sends an email after a week of signup if the user hasn't logged in" do
user = insert(:user, inserted_at: days_ago(8), last_seen: days_ago(8))
insert(:site, domain: "test-site.com", members: [user])
site = insert(:site, domain: "test-site.com", members: [user])
populate_stats(site, [build(:pageview)])
perform_job(SendCheckStatsEmails, %{})

12
test/workers/send_email_report_test.exs
View File

@ -39,17 +39,17 @@ defmodule Plausible.Workers.SendEmailReportTest do
create_pageviews([
# Sunday before last, not counted
%{domain: site.domain, timestamp: Timezone.convert(sunday_before_last, "UTC")},
%{site: site, timestamp: Timezone.convert(sunday_before_last, "UTC")},
# Sunday before last, not counted
%{domain: site.domain, timestamp: Timezone.convert(sunday_before_last, "UTC")},
%{site: site, timestamp: Timezone.convert(sunday_before_last, "UTC")},
# Last monday, counted
%{domain: site.domain, timestamp: Timezone.convert(last_monday, "UTC")},
%{site: site, timestamp: Timezone.convert(last_monday, "UTC")},
# Last sunday, counted
%{domain: site.domain, timestamp: Timezone.convert(last_sunday, "UTC")},
%{site: site, timestamp: Timezone.convert(last_sunday, "UTC")},
# This monday, not counted
%{domain: site.domain, timestamp: Timezone.convert(this_monday, "UTC")},
%{site: site, timestamp: Timezone.convert(this_monday, "UTC")},
# This monday, not counted
%{domain: site.domain, timestamp: Timezone.convert(this_monday, "UTC")}
%{site: site, timestamp: Timezone.convert(this_monday, "UTC")}
])
perform_job(SendEmailReport, %{"site_id" => site.id, "interval" => "weekly"})

6
test/workers/send_site_setup_emails_test.exs
View File

@ -41,7 +41,9 @@ defmodule Plausible.Workers.SendSiteSetupEmailsTest do
test "sends the setup completed email as soon as possible" do
user = insert(:user)
insert(:site, members: [user], domain: "test-site.com")
site = insert(:site, members: [user])
populate_stats(site, [build(:pageview)])
perform_job(SendSiteSetupEmails, %{})
@ -62,7 +64,7 @@ defmodule Plausible.Workers.SendSiteSetupEmailsTest do
subject: "Your Plausible setup: Waiting for the first page views"
)
create_pageviews([%{domain: site.domain}])
create_pageviews([%{site: site}])
perform_job(SendSiteSetupEmails, %{})
assert_email_delivered_with(

28
test/workers/send_trial_notifications_test.exs
View File

@ -44,7 +44,7 @@ defmodule Plausible.Workers.SendTrialNotificationsTest do
]
)
populate_stats(site, [build(:pageview, domain: site.domain)])
populate_stats(site, [build(:pageview)])
perform_job(SendTrialNotifications, %{})
@ -55,7 +55,7 @@ defmodule Plausible.Workers.SendTrialNotificationsTest do
test "sends a reminder 7 days before trial ends (16 days after user signed up)" do
user = insert(:user, trial_expiry_date: Timex.now() |> Timex.shift(days: 7))
site = insert(:site, members: [user])
populate_stats(site, [build(:pageview, domain: site.domain)])
populate_stats(site, [build(:pageview)])
perform_job(SendTrialNotifications, %{})
@ -67,9 +67,9 @@ defmodule Plausible.Workers.SendTrialNotificationsTest do
site = insert(:site, members: [user])
populate_stats(site, [
build(:pageview, domain: site.domain),
build(:pageview, domain: site.domain),
build(:pageview, domain: site.domain)
build(:pageview),
build(:pageview),
build(:pageview)
])
perform_job(SendTrialNotifications, %{})
@ -82,9 +82,9 @@ defmodule Plausible.Workers.SendTrialNotificationsTest do
site = insert(:site, members: [user])
populate_stats(site, [
build(:pageview, domain: site.domain),
build(:pageview, domain: site.domain),
build(:pageview, domain: site.domain)
build(:pageview),
build(:pageview),
build(:pageview)
])
perform_job(SendTrialNotifications, %{})
@ -115,9 +115,9 @@ defmodule Plausible.Workers.SendTrialNotificationsTest do
site = insert(:site, members: [user])
populate_stats(site, [
build(:pageview, domain: site.domain),
build(:pageview, domain: site.domain),
build(:pageview, domain: site.domain)
build(:pageview),
build(:pageview),
build(:pageview)
])
perform_job(SendTrialNotifications, %{})
@ -130,9 +130,9 @@ defmodule Plausible.Workers.SendTrialNotificationsTest do
site = insert(:site, members: [user])
populate_stats(site, [
build(:pageview, domain: site.domain),
build(:pageview, domain: site.domain),
build(:pageview, domain: site.domain)
build(:pageview),
build(:pageview),
build(:pageview)
])
insert(:subscription, user: user)