plausible/analytics
1
1
Fork 0
mirror of https://github.com/plausible/analytics.git synced 2024-11-26 00:24:44 +03:00
Code Issues Packages Projects Releases Wiki Activity

Switch to new clickhouse adapter (ch/chto) (#2733)

Browse Source 
* another clickhouse adapter

* don't restore stats_removal.ex

* fix events main-graph error (#2746)

* update ch, chto

* update chto again (#2759)

* Stop treating page filter as an entry_page filter (#2752)

* remove dead code

* stop treating page filter as entry page filter in breakdown queries

* stop treating page filter as entry page filter in aggregate queries

* stop treating page filter as entry page filter in timeseries queries

* mix format

* update changelog

* break code down to smaller functions to keep credo happy

* remove unused functions

* make CSV export return only conversions with goal filter (#2760)

* make CSV export return only conversions with goal filter

* update changelog

* update elixir version in mix.exs (#2742)

* revert admin.ex changes (#2776)

---------

Co-authored-by: ruslandoga <67764432+ruslandoga@users.noreply.github.com>
Co-authored-by: ruslandoga <rusl@n-do.ga>
Co-authored-by: RobertJoonas <56999674+RobertJoonas@users.noreply.github.com>
This commit is contained in:
Adam 2023-03-21 09:55:59 +01:00 committed by GitHub
parent 736e6e385c
commit 6d79ca5093
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
37 changed files with 782 additions and 178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
config/runtime.exs
View File

@ -296,7 +296,7 @@ config :plausible, Plausible.AsyncInsertRepo,
queue_interval: 2000,
url: ch_db_url,
pool_size: 1,
clickhouse_settings: [
settings: [
async_insert: 1,
wait_for_async_insert: 0
]

13
lib/mix/tasks/clean_clickhouse.ex
View File

@ -1,10 +1,15 @@
defmodule Mix.Tasks.CleanClickhouse do
use Mix.Task
alias Plausible.IngestRepo
def run(_) do
clean_events = "ALTER TABLE events DELETE WHERE 1"
clean_sessions = "ALTER TABLE sessions DELETE WHERE 1"
Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, clean_events)
Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, clean_sessions)
%{rows: rows} = IngestRepo.query!("show tables")
tables = Enum.map(rows, fn [table] -> table end)
to_truncate = tables -- ["schema_migrations"]
Enum.each(to_truncate, fn table ->
IngestRepo.query!("truncate #{table}")
end)
end
end

2
lib/plausible/async_insert_repo.ex
View File

@ -5,7 +5,7 @@ defmodule Plausible.AsyncInsertRepo do
use Ecto.Repo,
otp_app: :plausible,
adapter: ClickhouseEcto
adapter: Ecto.Adapters.ClickHouse
defmacro __using__(_) do
quote do

2
lib/plausible/clickhouse_repo.ex
View File

@ -1,7 +1,7 @@
defmodule Plausible.ClickhouseRepo do
use Ecto.Repo,
otp_app: :plausible,
adapter: ClickhouseEcto,
adapter: Ecto.Adapters.ClickHouse,
read_only: true
defmacro __using__(_) do

42
lib/plausible/event/clickhouse_schema.ex
View File

@ -8,32 +8,32 @@ defmodule Plausible.ClickhouseEvent do
field :domain, :string
field :hostname, :string
field :pathname, :string
field :user_id, :integer
field :session_id, :integer
field :user_id, Ch.Types.UInt64
field :session_id, Ch.Types.UInt64
field :timestamp, :naive_datetime
field :referrer, :string, default: ""
field :referrer_source, :string, default: ""
field :utm_medium, :string, default: ""
field :utm_source, :string, default: ""
field :utm_campaign, :string, default: ""
field :utm_content, :string, default: ""
field :utm_term, :string, default: ""
field :referrer, :string
field :referrer_source, :string
field :utm_medium, :string
field :utm_source, :string
field :utm_campaign, :string
field :utm_content, :string
field :utm_term, :string
field :country_code, :string, default: ""
field :subdivision1_code, :string, default: ""
field :subdivision2_code, :string, default: ""
field :city_geoname_id, :integer, default: 0
field :country_code, Ch.Types.FixedString, size: 2
field :subdivision1_code, :string
field :subdivision2_code, :string
field :city_geoname_id, Ch.Types.UInt32
field :screen_size, :string, default: ""
field :operating_system, :string, default: ""
field :operating_system_version, :string, default: ""
field :browser, :string, default: ""
field :browser_version, :string, default: ""
field :screen_size, :string
field :operating_system, :string
field :operating_system_version, :string
field :browser, :string
field :browser_version, :string
field :"meta.key", {:array, :string}, default: []
field :"meta.value", {:array, :string}, default: []
field :transferred_from, :string, default: ""
field :"meta.key", {:array, :string}
field :"meta.value", {:array, :string}
field :transferred_from, :string
end
def new(attrs) do

19
lib/plausible/google/buffer.ex
View File

@ -91,6 +91,23 @@ defmodule Plausible.Google.Buffer do
Process.sleep(1000)
Logger.info("Import: Flushing #{length(records)} from #{table_name} buffer")
Plausible.IngestRepo.insert_all(table_name, records)
insert_all(table_name, records)
end
# used in tests `setup`
@doc false
def insert_all(table_name, records) do
schema = table_schema(table_name)
Plausible.IngestRepo.insert_all(schema, records)
end
defp table_schema("imported_visitors"), do: Plausible.Google.ImportedVisitor
defp table_schema("imported_sources"), do: Plausible.Google.ImportedSource
defp table_schema("imported_pages"), do: Plausible.Google.ImportedPage
defp table_schema("imported_entry_pages"), do: Plausible.Google.ImportedEntryPage
defp table_schema("imported_exit_pages"), do: Plausible.Google.ImportedExitPage
defp table_schema("imported_locations"), do: Plausible.Google.ImportedLocation
defp table_schema("imported_devices"), do: Plausible.Google.ImportedDevice
defp table_schema("imported_browsers"), do: Plausible.Google.ImportedBrowser
defp table_schema("imported_operating_systems"), do: Plausible.Google.ImportedOperatingSystem
end

148
lib/plausible/google/schemas.ex Normal file
View File

@ -0,0 +1,148 @@
defmodule Plausible.Google.ImportedVisitor do
@moduledoc false
use Ecto.Schema
@primary_key false
schema "imported_visitors" do
field :site_id, Ch.Types.UInt64
field :date, :date
field :visitors, Ch.Types.UInt64
field :pageviews, Ch.Types.UInt64
field :bounces, Ch.Types.UInt64
field :visits, Ch.Types.UInt64
field :visit_duration, Ch.Types.UInt64
end
end
defmodule Plausible.Google.ImportedSource do
@moduledoc false
use Ecto.Schema
@primary_key false
schema "imported_sources" do
field :site_id, Ch.Types.UInt64
field :date, :date
field :source, :string
field :utm_medium, :string
field :utm_campaign, :string
field :utm_content, :string
field :utm_term, :string
field :visitors, Ch.Types.UInt64
field :visits, Ch.Types.UInt64
field :visit_duration, Ch.Types.UInt64
field :bounces, Ch.Types.UInt32
end
end
defmodule Plausible.Google.ImportedPage do
@moduledoc false
use Ecto.Schema
@primary_key false
schema "imported_pages" do
field :site_id, Ch.Types.UInt64
field :date, :date
field :hostname, :string
field :page, :string
field :visitors, Ch.Types.UInt64
field :pageviews, Ch.Types.UInt64
field :exits, Ch.Types.UInt64
field :time_on_page, Ch.Types.UInt64
end
end
defmodule Plausible.Google.ImportedEntryPage do
@moduledoc false
use Ecto.Schema
@primary_key false
schema "imported_entry_pages" do
field :site_id, Ch.Types.UInt64
field :date, :date
field :entry_page, :string
field :visitors, Ch.Types.UInt64
field :entrances, Ch.Types.UInt64
field :visit_duration, Ch.Types.UInt64
field :bounces, Ch.Types.UInt32
end
end
defmodule Plausible.Google.ImportedExitPage do
@moduledoc false
use Ecto.Schema
@primary_key false
schema "imported_exit_pages" do
field :site_id, Ch.Types.UInt64
field :date, :date
field :exit_page, :string
field :visitors, Ch.Types.UInt64
field :exits, Ch.Types.UInt64
end
end
defmodule Plausible.Google.ImportedLocation do
@moduledoc false
use Ecto.Schema
@primary_key false
schema "imported_locations" do
field :site_id, Ch.Types.UInt64
field :date, :date
field :country, :string
field :region, :string
field :city, Ch.Types.UInt64
field :visitors, Ch.Types.UInt64
field :visits, Ch.Types.UInt64
field :visit_duration, Ch.Types.UInt64
field :bounces, Ch.Types.UInt32
end
end
defmodule Plausible.Google.ImportedDevice do
@moduledoc false
use Ecto.Schema
@primary_key false
schema "imported_devices" do
field :site_id, Ch.Types.UInt64
field :date, :date
field :device, :string
field :visitors, Ch.Types.UInt64
field :visits, Ch.Types.UInt64
field :visit_duration, Ch.Types.UInt64
field :bounces, Ch.Types.UInt32
end
end
defmodule Plausible.Google.ImportedBrowser do
@moduledoc false
use Ecto.Schema
@primary_key false
schema "imported_browsers" do
field :site_id, Ch.Types.UInt64
field :date, :date
field :browser, :string
field :visitors, Ch.Types.UInt64
field :visits, Ch.Types.UInt64
field :visit_duration, Ch.Types.UInt64
field :bounces, Ch.Types.UInt32
end
end
defmodule Plausible.Google.ImportedOperatingSystem do
@moduledoc false
use Ecto.Schema
@primary_key false
schema "imported_operating_systems" do
field :site_id, Ch.Types.UInt64
field :date, :date
field :operating_system, :string
field :visitors, Ch.Types.UInt64
field :visits, Ch.Types.UInt64
field :visit_duration, Ch.Types.UInt64
field :bounces, Ch.Types.UInt32
end
end

2
lib/plausible/imported/site.ex
View File

@ -31,7 +31,7 @@ defmodule Plausible.Imported do
defp parse_number(nr) do
{float, ""} = Float.parse(nr)
float
round(float)
end
defp new_from_google_analytics(site_id, "imported_visitors", row) do

2
lib/plausible/ingest_repo.ex
View File

@ -5,7 +5,7 @@ defmodule Plausible.IngestRepo do
use Ecto.Repo,
otp_app: :plausible,
adapter: ClickhouseEcto
adapter: Ecto.Adapters.ClickHouse
defmacro __using__(_) do
quote do

2
lib/plausible/ingestion/counters.ex
View File

@ -14,7 +14,7 @@ defmodule Plausible.Ingestion.Counters do
aggregate into a 1-minute resolution.
Clickhouse connection is set to insert counters asynchronously every time
a pool checkout is made. Those properties are reverted once the insert is done
a pool checkout is made. Those properties are reverted once the insert is done
(or naturally, if the connection crashes).
"""

4
lib/plausible/ingestion/counters/record.ex
View File

@ -9,9 +9,9 @@ defmodule Plausible.Ingestion.Counters.Record do
@primary_key false
schema "ingest_counters" do
field :event_timebucket, :utc_datetime
field :site_id, :integer
field :site_id, Ch.Types.Nullable, type: Ch.Types.UInt64
field :domain, :string
field :metric, :string
field :value, :integer
field :value, Ch.Types.UInt64
end
end

2
lib/plausible/purge.ex
View File

@ -16,7 +16,7 @@ defmodule Plausible.Purge do
"""
def delete_imported_stats!(site) do
Enum.each(Plausible.Imported.tables(), fn table ->
sql = "ALTER TABLE #{table} DELETE WHERE site_id = ?"
sql = "ALTER TABLE #{table} DELETE WHERE site_id = {$0:UInt64}"
Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, sql, [site.id])
end)

43
lib/plausible/session/cache_store.ex
View File

@ -53,41 +53,16 @@ defmodule Plausible.Session.CacheStore do
duration: Timex.diff(event.timestamp, session.start, :second) |> abs,
pageviews:
if(event.name == "pageview", do: session.pageviews + 1, else: session.pageviews),
country_code:
if(session.country_code == "", do: event.country_code, else: session.country_code),
subdivision1_code:
if(session.subdivision1_code == "",
do: event.subdivision1_code,
else: session.subdivision1_code
),
subdivision2_code:
if(session.subdivision2_code == "",
do: event.subdivision2_code,
else: session.subdivision2_code
),
city_geoname_id:
if(session.city_geoname_id == 0,
do: event.city_geoname_id,
else: session.city_geoname_id
),
operating_system:
if(session.operating_system == "",
do: event.operating_system,
else: session.operating_system
),
country_code: session.country_code || event.country_code,
subdivision1_code: session.subdivision1_code || event.subdivision1_code,
subdivision2_code: session.subdivision2_code || event.subdivision2_code,
city_geoname_id: session.city_geoname_id || event.city_geoname_id,
operating_system: session.operating_system || event.operating_system,
operating_system_version:
if(session.operating_system_version == "",
do: event.operating_system_version,
else: session.operating_system_version
),
browser: if(session.browser == "", do: event.browser, else: session.browser),
browser_version:
if(session.browser_version == "",
do: event.browser_version,
else: session.browser_version
),
screen_size:
if(session.screen_size == "", do: event.screen_size, else: session.screen_size),
session.operating_system_version || event.operating_system_version,
browser: session.browser || event.browser,
browser_version: session.browser_version || event.browser_version,
screen_size: session.screen_size || event.screen_size,
events: session.events + 1
}
end

26
lib/plausible/session/clickhouse_schema.ex
View File

@ -6,20 +6,20 @@ defmodule Plausible.ClickhouseSession do
schema "sessions" do
field :hostname, :string
field :domain, :string
field :user_id, :integer
field :session_id, :integer
field :user_id, Ch.Types.UInt64
field :session_id, Ch.Types.UInt64
field :start, :naive_datetime
field :duration, :integer
field :duration, Ch.Types.UInt32
field :is_bounce, :boolean
field :entry_page, :string
field :exit_page, :string
field :pageviews, :integer
field :events, :integer
field :sign, :integer
field :pageviews, Ch.Types.Int32
field :events, Ch.Types.Int32
field :sign, Ch.Types.Int8
field :"entry_meta.key", {:array, :string}, default: []
field :"entry_meta.value", {:array, :string}, default: []
field :"entry_meta.key", {:array, :string}
field :"entry_meta.value", {:array, :string}
field :utm_medium, :string
field :utm_source, :string
@ -29,10 +29,10 @@ defmodule Plausible.ClickhouseSession do
field :referrer, :string
field :referrer_source, :string
field :country_code, :string, default: ""
field :subdivision1_code, :string, default: ""
field :subdivision2_code, :string, default: ""
field :city_geoname_id, :integer, default: 0
field :country_code, Ch.Types.FixedString, size: 2
field :subdivision1_code, :string
field :subdivision2_code, :string
field :city_geoname_id, Ch.Types.UInt32
field :screen_size, :string
field :operating_system, :string
@ -41,7 +41,7 @@ defmodule Plausible.ClickhouseSession do
field :browser_version, :string
field :timestamp, :naive_datetime
field :transferred_from, :string, default: ""
field :transferred_from, :string
end
def random_uint64() do

14
lib/plausible/stats/aggregate.ex
View File

@ -61,24 +61,28 @@ defmodule Plausible.Stats.Aggregate do
)
{base_query_raw, base_query_raw_params} = ClickhouseRepo.to_sql(:all, q)
where_param_idx = length(base_query_raw_params)
where_param = "{$#{where_param_idx}:String}"
{where_clause, where_arg} =
case query.filters["event:page"] do
{:is, page} ->
{"p = ?", page}
{"p = #{where_param}", page}
{:is_not, page} ->
{"p != ?", page}
{"p != #{where_param}", page}
{:matches, expr} ->
regex = page_regex(expr)
{"match(p, ?)", regex}
{"match(p, #{where_param})", regex}
{:does_not_match, expr} ->
regex = page_regex(expr)
{"not(match(p, ?))", regex}
{"not(match(p, #{where_param}))", regex}
end
params = base_query_raw_params ++ [where_arg]
time_query = "
SELECT
avg(ifNotFinite(avgTime, null))
@ -102,7 +106,7 @@ defmodule Plausible.Stats.Aggregate do
GROUP BY p,p2,s)
GROUP BY p)"
{:ok, res} = ClickhouseRepo.query(time_query, base_query_raw_params ++ [where_arg])
{:ok, res} = ClickhouseRepo.query(time_query, params)
[[time_on_page]] = res.rows
%{time_on_page: time_on_page}
end

10
lib/plausible/stats/base.ex
View File

@ -174,7 +174,7 @@ defmodule Plausible.Stats.Base do
{:member, values} ->
list = Enum.map(values, &db_prop_val(prop_name, &1))
from(s in sessions_q, where: fragment("? in tuple(?)", field(s, ^prop_name), ^list))
from(s in sessions_q, where: field(s, ^prop_name) in ^list)
{:matches, expr} ->
regex = page_regex(expr)
@ -399,8 +399,10 @@ defmodule Plausible.Stats.Base do
NaiveDateTime.utc_now()
|> Timex.shift(seconds: 5)
|> beginning_of_time(site.native_stats_start_at)
|> NaiveDateTime.truncate(:second)
first_datetime = NaiveDateTime.utc_now() |> Timex.shift(minutes: -5)
first_datetime =
NaiveDateTime.utc_now() |> Timex.shift(minutes: -5) |> NaiveDateTime.truncate(:second)
{first_datetime, last_datetime}
end
@ -410,8 +412,10 @@ defmodule Plausible.Stats.Base do
NaiveDateTime.utc_now()
|> Timex.shift(seconds: 5)
|> beginning_of_time(site.native_stats_start_at)
|> NaiveDateTime.truncate(:second)
first_datetime = NaiveDateTime.utc_now() |> Timex.shift(minutes: -30)
first_datetime =
NaiveDateTime.utc_now() |> Timex.shift(minutes: -30) |> NaiveDateTime.truncate(:second)
{first_datetime, last_datetime}
end

13
lib/plausible/stats/breakdown.ex
View File

@ -44,14 +44,14 @@ defmodule Plausible.Stats.Breakdown do
offset: ^offset,
where:
fragment(
"notEmpty(multiMatchAllIndices(?, array(?)) as indices)",
"notEmpty(multiMatchAllIndices(?, ?) as indices)",
e.pathname,
^page_regexes
),
group_by: fragment("index"),
select: %{
index: fragment("arrayJoin(indices) as index"),
goal: fragment("concat('Visit ', array(?)[index])", ^page_exprs)
goal: fragment("concat('Visit ', ?[index])", ^page_exprs)
}
)
|> select_event_metrics(metrics)
@ -242,6 +242,9 @@ defmodule Plausible.Stats.Breakdown do
"round(sum(td)/count(case when p2 != p then 1 end))"
end
pages_idx = length(base_query_raw_params)
params = base_query_raw_params ++ [pages]
time_query = "
SELECT
p,
@ -258,11 +261,11 @@ defmodule Plausible.Stats.Breakdown do
neighbor(p, 1) as p2,
neighbor(s, 1) as s2
FROM (#{base_query_raw}))
WHERE s=s2 AND p IN tuple(?)
WHERE s=s2 AND p IN {$#{pages_idx}:Array(String)}
GROUP BY p,p2,s)
GROUP BY p"
{:ok, res} = ClickhouseRepo.query(time_query, base_query_raw_params ++ [pages])
{:ok, res} = ClickhouseRepo.query(time_query, params)
if query.include_imported do
# Imported page views have pre-calculated values
@ -367,7 +370,7 @@ defmodule Plausible.Stats.Breakdown do
group_by: fragment("index"),
select_merge: %{
index: fragment("arrayJoin(indices) as index"),
page_match: fragment("array(?)[index]", ^match_exprs)
page_match: fragment("?[index]", ^match_exprs)
},
order_by: {:asc, fragment("index")}
)

18
lib/plausible/stats/clickhouse.ex
View File

@ -176,7 +176,7 @@ defmodule Plausible.Stats.Clickhouse do
ClickhouseRepo.all(
from e in "events",
group_by: e.domain,
where: fragment("? IN tuple(?)", e.domain, ^domains),
where: e.domain in ^domains,
where: e.timestamp > fragment("now() - INTERVAL 24 HOUR"),
select: {e.domain, fragment("uniq(user_id)")}
)
@ -421,7 +421,7 @@ defmodule Plausible.Stats.Clickhouse do
else
from(
e in q,
inner_lateral_join: meta in fragment("meta as m"),
inner_lateral_join: meta in fragment("meta"),
where: meta.key == ^key and meta.value == ^val
)
end
@ -435,19 +435,25 @@ defmodule Plausible.Stats.Clickhouse do
end
defp utc_boundaries(%Query{period: "30m"}, site) do
last_datetime = NaiveDateTime.utc_now()
last_datetime = NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second)
first_datetime =
last_datetime |> Timex.shift(minutes: -30) |> beginning_of_time(site.native_stats_start_at)
last_datetime
|> Timex.shift(minutes: -30)
|> beginning_of_time(site.native_stats_start_at)
|> NaiveDateTime.truncate(:second)
{first_datetime, last_datetime}
end
defp utc_boundaries(%Query{period: "realtime"}, site) do
last_datetime = NaiveDateTime.utc_now()
last_datetime = NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second)
first_datetime =
last_datetime |> Timex.shift(minutes: -5) |> beginning_of_time(site.native_stats_start_at)
last_datetime
|> Timex.shift(minutes: -5)
|> beginning_of_time(site.native_stats_start_at)
|> NaiveDateTime.truncate(:second)
{first_datetime, last_datetime}
end

1
lib/plausible/stats/current_visitors.ex
View File

@ -6,6 +6,7 @@ defmodule Plausible.Stats.CurrentVisitors do
first_datetime =
NaiveDateTime.utc_now()
|> Timex.shift(minutes: -5)
|> NaiveDateTime.truncate(:second)
ClickhouseRepo.one(
from e in "events",

4
lib/plausible/stats/imported.ex
View File

@ -27,8 +27,8 @@ defmodule Plausible.Stats.Imported do
from(s in Ecto.Query.subquery(native_q),
full_join: i in subquery(imported_q),
on: field(s, :date) == field(i, :date),
select: %{date: field(s, :date)}
on: s.date == i.date,
select: %{date: fragment("greatest(?, ?)", s.date, i.date)}
)
|> select_joined_metrics(metrics)
end

2
lib/plausible/stats/props.ex
View File

@ -51,7 +51,7 @@ defmodule Plausible.Stats.Props do
nil ->
ClickhouseRepo.all(
from e in base_event_query(site, query),
inner_lateral_join: meta in fragment("meta as m"),
inner_lateral_join: meta in fragment("meta"),
select: {e.name, meta.key},
distinct: true
)

27
lib/plausible/stats/timeseries.ex
View File

@ -24,8 +24,9 @@ defmodule Plausible.Stats.Timeseries do
Enum.map(steps, fn step ->
empty_row(step, metrics)
|> Map.merge(Enum.find(event_result, fn row -> row[:date] == step end) || %{})
|> Map.merge(Enum.find(session_result, fn row -> row[:date] == step end) || %{})
|> Map.merge(Enum.find(event_result, fn row -> date_eq(row[:date], step) end) || %{})
|> Map.merge(Enum.find(session_result, fn row -> date_eq(row[:date], step) end) || %{})
|> Map.update!(:date, &date_format/1)
end)
end
@ -88,7 +89,6 @@ defmodule Plausible.Stats.Timeseries do
query.date_range.first
|> Timex.to_datetime()
|> Timex.shift(hours: step)
|> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{m}:{s}")
end)
end
@ -109,10 +109,29 @@ defmodule Plausible.Stats.Timeseries do
query.date_range.first
|> Timex.to_datetime()
|> Timex.shift(minutes: step)
|> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{m}:{s}")
end)
end
defp date_eq(%DateTime{} = left, %DateTime{} = right) do
NaiveDateTime.compare(left, right) == :eq
end
defp date_eq(%Date{} = left, %Date{} = right) do
Date.compare(left, right) == :eq
end
defp date_eq(left, right) do
left == right
end
defp date_format(%DateTime{} = date) do
Timex.format!(date, "{YYYY}-{0M}-{0D} {h24}:{m}:{s}")
end
defp date_format(date) do
date
end
defp select_bucket(q, site, %Query{interval: "month"}) do
from(
e in q,

3
mix.exs
View File

@ -62,8 +62,7 @@ defmodule Plausible.MixProject do
{:bcrypt_elixir, "~> 2.0"},
{:bypass, "~> 2.1", only: [:dev, :test]},
{:cachex, "~> 3.4"},
{:clickhouse_ecto, git: "https://github.com/plausible/clickhouse_ecto.git"},
{:clickhousex, github: "plausible/clickhousex", override: true},
{:chto, github: "ruslandoga/chto"},
{:combination, "~> 0.0.3"},
{:connection, "~> 1.1", override: true},
{:cors_plug, "~> 3.0"},

10
mix.lock
View File

@ -8,11 +8,11 @@
"bunt": {:hex, :bunt, "0.2.1", "e2d4792f7bc0ced7583ab54922808919518d0e57ee162901a16a1b6664ef3b14", [:mix], [], "hexpm", "a330bfb4245239787b15005e66ae6845c9cd524a288f0d141c148b02603777a5"},
"bypass": {:hex, :bypass, "2.1.0", "909782781bf8e20ee86a9cabde36b259d44af8b9f38756173e8f5e2e1fabb9b1", [:mix], [{:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "d9b5df8fa5b7a6efa08384e9bbecfe4ce61c77d28a4282f79e02f1ef78d96b80"},
"cachex": {:hex, :cachex, "3.4.0", "868b2959ea4aeb328c6b60ff66c8d5123c083466ad3c33d3d8b5f142e13101fb", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "370123b1ab4fba4d2965fb18f87fd758325709787c8c5fce35b3fe80645ccbe5"},
"castore": {:hex, :castore, "0.1.20", "62a0126cbb7cb3e259257827b9190f88316eb7aa3fdac01fd6f2dfd64e7f46e9", [:mix], [], "hexpm", "a020b7650529c986c454a4035b6b13a328e288466986307bea3aadb4c95ac98a"},
"castore": {:hex, :castore, "0.1.22", "4127549e411bedd012ca3a308dede574f43819fe9394254ca55ab4895abfa1a2", [:mix], [], "hexpm", "c17576df47eb5aa1ee40cc4134316a99f5cad3e215d5c77b8dd3cfef12a22cac"},
"certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"},
"chatterbox": {:hex, :ts_chatterbox, "0.13.0", "6f059d97bcaa758b8ea6fffe2b3b81362bd06b639d3ea2bb088335511d691ebf", [:rebar3], [{:hpack, "~>0.2.3", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "b93d19104d86af0b3f2566c4cba2a57d2e06d103728246ba1ac6c3c0ff010aa7"},
"clickhouse_ecto": {:git, "https://github.com/plausible/clickhouse_ecto.git", "43126c020f07b097c55a81f68a906019fd061f29", []},
"clickhousex": {:git, "https://github.com/plausible/clickhousex.git", "cfeefaf21f0841bc4e5adccb3b1875a1e5eb3ee7", []},
"ch": {:git, "https://github.com/ruslandoga/ch.git", "5894f81e246c049b9db5186497189cce226947aa", []},
"chatterbox": {:hex, :ts_chatterbox, "0.13.0", "6f059d97bcaa758b8ea6fffe2b3b81362bd06b639d3ea2bb088335511d691ebf", [:rebar3], [{:hpack, "~> 0.2.3", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "b93d19104d86af0b3f2566c4cba2a57d2e06d103728246ba1ac6c3c0ff010aa7"},
"chto": {:git, "https://github.com/ruslandoga/chto.git", "b5913b36331d5bb0a349b08b592da42410643cd9", []},
"combination": {:hex, :combination, "0.0.3", "746aedca63d833293ec6e835aa1f34974868829b1486b1e1cb0685f0b2ae1f41", [:mix], [], "hexpm", "72b099f463df42ef7dc6371d250c7070b57b6c5902853f69deb894f79eda18ca"},
"combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},
"comeonin": {:hex, :comeonin, "5.3.3", "2c564dac95a35650e9b6acfe6d2952083d8a08e4a89b93a481acb552b325892e", [:mix], [], "hexpm", "3e38c9c2cb080828116597ca8807bb482618a315bfafd98c90bc22a821cc84df"},
@ -32,7 +32,6 @@
"ecto": {:hex, :ecto, "3.9.4", "3ee68e25dbe0c36f980f1ba5dd41ee0d3eb0873bccae8aeaf1a2647242bffa35", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "de5f988c142a3aa4ec18b85a4ec34a2390b65b24f02385c1144252ff6ff8ee75"},
"ecto_sql": {:hex, :ecto_sql, "3.9.2", "34227501abe92dba10d9c3495ab6770e75e79b836d114c41108a4bf2ce200ad5", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.9.2", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "1eb5eeb4358fdbcd42eac11c1fbd87e3affd7904e639d77903c1358b2abd3f70"},
"elixir_make": {:hex, :elixir_make, "0.6.3", "bc07d53221216838d79e03a8019d0839786703129599e9619f4ab74c8c096eac", [:mix], [], "hexpm", "f5cbd651c5678bcaabdbb7857658ee106b12509cd976c2c2fca99688e1daf716"},
"elixir_uuid": {:hex, :elixir_uuid, "1.2.1", "dce506597acb7e6b0daeaff52ff6a9043f5919a4c3315abb4143f0b00378c097", [:mix], [], "hexpm", "f7eba2ea6c3555cea09706492716b0d87397b88946e6380898c2889d68585752"},
"envy": {:hex, :envy, "1.1.1", "0bc9bd654dec24fcdf203f7c5aa1b8f30620f12cfb28c589d5e9c38fe1b07475", [:mix], [], "hexpm", "7061eb1a47415fd757145d8dec10dc0b1e48344960265cb108f194c4252c3a89"},
"erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"},
"eternal": {:hex, :eternal, "1.2.2", "d1641c86368de99375b98d183042dd6c2b234262b8d08dfd72b9eeaafc2a1abd", [:mix], [], "hexpm", "2c9fe32b9c3726703ba5e1d43a1d255a4f3f2d8f8f9bc19f094c7cb1a7a9e782"},
@ -58,7 +57,6 @@
"hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"},
"html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"},
"httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"},
"hut": {:hex, :hut, "1.3.0", "71f2f054e657c03f959cf1acc43f436ea87580696528ca2a55c8afb1b06c85e7", [:"erlang.mk", :rebar, :rebar3], [], "hexpm", "7e15d28555d8a1f2b5a3a931ec120af0753e4853a4c66053db354f35bf9ab563"},
"idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
"jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"},
"jsx": {:hex, :jsx, "2.8.3", "a05252d381885240744d955fbe3cf810504eb2567164824e19303ea59eef62cf", [:mix, :rebar3], [], "hexpm", "fc3499fed7a726995aa659143a248534adc754ebd16ccd437cd93b649a95091f"},

4
priv/ingest_repo/migrations/.formatter.exs Normal file
View File

@ -0,0 +1,4 @@
[
import_deps: [:ecto_sql],
inputs: ["*.exs"]
]

14
priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs
View File

@ -8,8 +8,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateEventsAndSessions do
defp create_events() do
create_if_not_exists table(:events,
engine:
"MergeTree() PARTITION BY toYYYYMM(timestamp) ORDER BY (domain, toDate(timestamp), user_id) SETTINGS index_granularity = 8192"
primary_key: false,
engine: "MergeTree",
options:
"PARTITION BY toYYYYMM(timestamp) ORDER BY (domain, toDate(timestamp), user_id) SETTINGS index_granularity = 8192"
) do
add(:name, :string)
add(:domain, :string)
@ -30,15 +32,17 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateEventsAndSessions do
defp create_sessions() do
create_if_not_exists table(:sessions,
engine:
"CollapsingMergeTree(sign) PARTITION BY toYYYYMM(start) ORDER BY (domain, toDate(start), user_id, session_id) SETTINGS index_granularity = 8192"
primary_key: false,
engine: "CollapsingMergeTree(sign)",
options:
"PARTITION BY toYYYYMM(start) ORDER BY (domain, toDate(start), user_id, session_id) SETTINGS index_granularity = 8192"
) do
add(:session_id, :UInt64)
add(:sign, :Int8)
add(:domain, :string)
add(:user_id, :UInt64)
add(:hostname, :string)
add(:is_bounce, :boolean)
add(:is_bounce, :UInt8)
add(:entry_page, :string)
add(:exit_page, :string)
add(:pageviews, :integer)

2
priv/ingest_repo/migrations/20201020083739_add_event_metadata.exs
View File

@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.AddEventMetadata do
def change do
alter table(:events) do
add :meta, {:nested, {{:key, :string}, {:value, :string}}}
add :meta, :"Nested(key String, value String)"
end
end
end

36
priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs
View File

@ -3,7 +3,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
def change do
create_if_not_exists table(:imported_visitors,
engine: "MergeTree() ORDER BY (site_id, date)"
primary_key: false,
engine: "MergeTree",
options: "ORDER BY (site_id, date)"
) do
add(:site_id, :UInt64)
add(:date, :date)
@ -15,7 +17,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
end
create_if_not_exists table(:imported_sources,
engine: "MergeTree() ORDER BY (site_id, date, source)"
primary_key: false,
engine: "MergeTree",
options: "ORDER BY (site_id, date, source)"
) do
add(:site_id, :UInt64)
add(:date, :date)
@ -31,7 +35,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
end
create_if_not_exists table(:imported_pages,
engine: "MergeTree() ORDER BY (site_id, date, hostname, page)"
primary_key: false,
engine: "MergeTree",
options: "ORDER BY (site_id, date, hostname, page)"
) do
add(:site_id, :UInt64)
add(:date, :date)
@ -44,7 +50,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
end
create_if_not_exists table(:imported_entry_pages,
engine: "MergeTree() ORDER BY (site_id, date, entry_page)"
primary_key: false,
engine: "MergeTree",
options: "ORDER BY (site_id, date, entry_page)"
) do
add(:site_id, :UInt64)
add(:date, :date)
@ -56,7 +64,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
end
create_if_not_exists table(:imported_exit_pages,
engine: "MergeTree() ORDER BY (site_id, date, exit_page)"
primary_key: false,
engine: "MergeTree",
options: "ORDER BY (site_id, date, exit_page)"
) do
add(:site_id, :UInt64)
add(:date, :date)
@ -66,7 +76,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
end
create_if_not_exists table(:imported_locations,
engine: "MergeTree() ORDER BY (site_id, date, country, region, city)"
primary_key: false,
engine: "MergeTree",
options: "ORDER BY (site_id, date, country, region, city)"
) do
add(:site_id, :UInt64)
add(:date, :date)
@ -80,7 +92,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
end
create_if_not_exists table(:imported_devices,
engine: "MergeTree() ORDER BY (site_id, date, device)"
primary_key: false,
engine: "MergeTree",
options: "ORDER BY (site_id, date, device)"
) do
add(:site_id, :UInt64)
add(:date, :date)
@ -92,7 +106,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
end
create_if_not_exists table(:imported_browsers,
engine: "MergeTree() ORDER BY (site_id, date, browser)"
primary_key: false,
engine: "MergeTree",
options: "ORDER BY (site_id, date, browser)"
) do
add(:site_id, :UInt64)
add(:date, :date)
@ -104,7 +120,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
end
create_if_not_exists table(:imported_operating_systems,
engine: "MergeTree() ORDER BY (site_id, date, operating_system)"
primary_key: false,
engine: "MergeTree",
options: "ORDER BY (site_id, date, operating_system)"
) do
add(:site_id, :UInt64)
add(:date, :date)

2
priv/ingest_repo/migrations/20220404123000_add_entry_props_to_session.exs
View File

@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.AddEntryPropsToSession do
def change do
alter table(:sessions) do
add :"entry.meta", {:nested, {{:key, :string}, {:value, :string}}}
add :"entry.meta", :"Nested(key String, value String)"
end
end
end

2
priv/ingest_repo/migrations/20220422075510_add_entry_props.exs
View File

@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.AddEntryProps do
def change do
alter table(:sessions) do
add(:entry_meta, {:nested, {{:key, :string}, {:value, :string}}})
add(:entry_meta, :"Nested(key String, value String)")
end
end
end

7
priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs
View File

@ -3,8 +3,11 @@ defmodule Plausible.IngestRepo.Migrations.CreateIngestCountersTable do
def change do
create_if_not_exists table(:ingest_counters,
engine: "SummingMergeTree(value) ORDER BY (domain, toDate(event_timebucket), metric, toStartOfMinute(event_timebucket))") do
primary_key: false,
engine: "SummingMergeTree(value)",
options:
"ORDER BY (domain, toDate(event_timebucket), metric, toStartOfMinute(event_timebucket))"
) do
add(:event_timebucket, :utc_datetime)
add(:domain, :"LowCardinality(String)")
add(:site_id, :"Nullable(UInt64)")

242
priv/ingest_repo/structure.sql Normal file
View File

@ -0,0 +1,242 @@
CREATE TABLE plausible_events_db.sessions
(
`session_id` UInt64,
`sign` Int8,
`domain` String,
`user_id` UInt64,
`hostname` String,
`is_bounce` UInt8,
`entry_page` String,
`exit_page` String,
`pageviews` Int32,
`events` Int32,
`duration` UInt32,
`referrer` String,
`referrer_source` String,
`country_code` LowCardinality(FixedString(2)),
`screen_size` LowCardinality(String),
`operating_system` LowCardinality(String),
`browser` LowCardinality(String),
`start` DateTime,
`timestamp` DateTime,
`utm_medium` String,
`utm_source` String,
`utm_campaign` String,
`browser_version` LowCardinality(String),
`operating_system_version` LowCardinality(String),
`subdivision1_code` LowCardinality(String),
`subdivision2_code` LowCardinality(String),
`city_geoname_id` UInt32,
`utm_content` String,
`utm_term` String,
`transferred_from` String,
`entry_meta.key` Array(String),
`entry_meta.value` Array(String)
)
ENGINE = CollapsingMergeTree(sign)
PARTITION BY toYYYYMM(start)
ORDER BY (domain, toDate(start), user_id, session_id)
SAMPLE BY user_id
SETTINGS index_granularity = 8192;
CREATE TABLE plausible_events_db.ingest_counters
(
`event_timebucket` DateTime,
`domain` LowCardinality(String),
`site_id` Nullable(UInt64),
`metric` LowCardinality(String),
`value` UInt64
)
ENGINE = SummingMergeTree(value)
ORDER BY (domain, toDate(event_timebucket), metric, toStartOfMinute(event_timebucket))
SETTINGS index_granularity = 8192;
CREATE TABLE plausible_events_db.imported_visitors
(
`site_id` UInt64,
`date` Date,
`visitors` UInt64,
`pageviews` UInt64,
`bounces` UInt64,
`visits` UInt64,
`visit_duration` UInt64
)
ENGINE = MergeTree
ORDER BY (site_id, date)
SETTINGS index_granularity = 8192;
CREATE TABLE plausible_events_db.imported_sources
(
`site_id` UInt64,
`date` Date,
`source` String,
`utm_medium` String,
`utm_campaign` String,
`utm_content` String,
`utm_term` String,
`visitors` UInt64,
`visits` UInt64,
`visit_duration` UInt64,
`bounces` UInt32
)
ENGINE = MergeTree
ORDER BY (site_id, date, source)
SETTINGS index_granularity = 8192;
CREATE TABLE plausible_events_db.imported_pages
(
`site_id` UInt64,
`date` Date,
`hostname` String,
`page` String,
`visitors` UInt64,
`pageviews` UInt64,
`exits` UInt64,
`time_on_page` UInt64
)
ENGINE = MergeTree
ORDER BY (site_id, date, hostname, page)
SETTINGS index_granularity = 8192;
CREATE TABLE plausible_events_db.imported_operating_systems
(
`site_id` UInt64,
`date` Date,
`operating_system` String,
`visitors` UInt64,
`visits` UInt64,
`visit_duration` UInt64,
`bounces` UInt32
)
ENGINE = MergeTree
ORDER BY (site_id, date, operating_system)
SETTINGS index_granularity = 8192;
CREATE TABLE plausible_events_db.imported_locations
(
`site_id` UInt64,
`date` Date,
`country` String,
`region` String,
`city` UInt64,
`visitors` UInt64,
`visits` UInt64,
`visit_duration` UInt64,
`bounces` UInt32
)
ENGINE = MergeTree
ORDER BY (site_id, date, country, region, city)
SETTINGS index_granularity = 8192;
CREATE TABLE plausible_events_db.imported_exit_pages
(
`site_id` UInt64,
`date` Date,
`exit_page` String,
`visitors` UInt64,
`exits` UInt64
)
ENGINE = MergeTree
ORDER BY (site_id, date, exit_page)
SETTINGS index_granularity = 8192;
CREATE TABLE plausible_events_db.imported_entry_pages
(
`site_id` UInt64,
`date` Date,
`entry_page` String,
`visitors` UInt64,
`entrances` UInt64,
`visit_duration` UInt64,
`bounces` UInt32
)
ENGINE = MergeTree
ORDER BY (site_id, date, entry_page)
SETTINGS index_granularity = 8192;
CREATE TABLE plausible_events_db.imported_devices
(
`site_id` UInt64,
`date` Date,
`device` String,
`visitors` UInt64,
`visits` UInt64,
`visit_duration` UInt64,
`bounces` UInt32
)
ENGINE = MergeTree
ORDER BY (site_id, date, device)
SETTINGS index_granularity = 8192;
CREATE TABLE plausible_events_db.imported_browsers
(
`site_id` UInt64,
`date` Date,
`browser` String,
`visitors` UInt64,
`visits` UInt64,
`visit_duration` UInt64,
`bounces` UInt32
)
ENGINE = MergeTree
ORDER BY (site_id, date, browser)
SETTINGS index_granularity = 8192;
CREATE TABLE plausible_events_db.events
(
`name` String,
`domain` String,
`user_id` UInt64,
`session_id` UInt64,
`hostname` String,
`pathname` String,
`referrer` String,
`referrer_source` String,
`country_code` LowCardinality(FixedString(2)),
`screen_size` LowCardinality(String),
`operating_system` LowCardinality(String),
`browser` LowCardinality(String),
`timestamp` DateTime,
`utm_medium` String,
`utm_source` String,
`utm_campaign` String,
`meta.key` Array(String),
`meta.value` Array(String),
`browser_version` LowCardinality(String),
`operating_system_version` LowCardinality(String),
`subdivision1_code` LowCardinality(String),
`subdivision2_code` LowCardinality(String),
`city_geoname_id` UInt32,
`utm_content` String,
`utm_term` String,
`transferred_from` String
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(timestamp)
ORDER BY (domain, toDate(timestamp), user_id)
SAMPLE BY user_id
SETTINGS index_granularity = 8192;
CREATE TABLE plausible_events_db.schema_migrations
(
`version` Int64,
`inserted_at` DateTime
)
ENGINE = TinyLog;
INSERT INTO "plausible_events_db"."schema_migrations" (version, inserted_at) VALUES
(20200915070607,'2023-03-08 10:03:33'),
(20200918075025,'2023-03-08 10:03:33'),
(20201020083739,'2023-03-08 10:03:33'),
(20201106125234,'2023-03-08 10:03:33'),
(20210323130440,'2023-03-08 10:03:33'),
(20210712214034,'2023-03-08 10:03:33'),
(20211017093035,'2023-03-08 10:03:33'),
(20211112130238,'2023-03-08 10:03:33'),
(20220310104931,'2023-03-08 10:03:33'),
(20220404123000,'2023-03-08 10:03:33'),
(20220421161259,'2023-03-08 10:03:33'),
(20220422075510,'2023-03-08 10:03:33'),
(20230124140348,'2023-03-08 10:03:33'),
(20230210140348,'2023-03-08 10:03:33'),
(20230214114402,'2023-03-08 10:03:33');

2
test/plausible/imported/imported_test.exs
View File

@ -7,7 +7,7 @@ defmodule Plausible.ImportedTest do
defp import_data(ga_data, site_id, table_name) do
ga_data
|> Plausible.Imported.from_google_analytics(site_id, table_name)
|> then(&Plausible.IngestRepo.insert_all(table_name, &1))
|> then(&Plausible.Google.Buffer.insert_all(table_name, &1))
end
describe "Parse and import third party data fetched from Google Analytics" do

4
test/plausible/ingestion/counters_test.exs
View File

@ -37,8 +37,8 @@ defmodule Plausible.Ingestion.CountersTest do
end
test "the database eventually sums the records within 1-minute buckets", %{test: test} do
# Testing if the database works is an unfunny way of integration testing,
# but on the upside it's quite straight-forward way of testing if the
# Testing if the database works is an unfunny way of integration testing,
# but on the upside it's quite straight-forward way of testing if the
# 1-minute bucket rollups are applied when dumping the records that are
# originally aggregated with 10s windows.
on_exit(:detach, fn ->

157
test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
View File

@ -39,6 +39,25 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert plot == [1] ++ zeroes ++ [1]
end
test "displays visitors for a day with imported data", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
build(:pageview, timestamp: ~N[2021-01-31 00:00:00]),
build(:imported_visitors, date: ~D[2021-01-01]),
build(:imported_visitors, date: ~D[2021-01-31])
])
conn =
get(
conn,
"/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&with_imported=true"
)
assert %{"plot" => plot, "imported_source" => "Google Analytics"} = json_response(conn, 200)
assert plot == [2] ++ List.duplicate(0, 23)
end
test "displays hourly stats in configured timezone", %{conn: conn, user: user} do
# UTC+1
site =
@ -60,7 +79,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert %{"plot" => plot} = json_response(conn, 200)
zeroes = Stream.repeatedly(fn -> 0 end) |> Stream.take(22) |> Enum.into([])
zeroes = List.duplicate(0, 22)
# Expecting pageview to show at 1am CET
assert plot == [0, 1] ++ zeroes
@ -108,6 +127,26 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert Enum.sum(plot) == 4
end
test "displays visitors for a month with only imported data", %{conn: conn, site: site} do
populate_stats(site, [
build(:imported_visitors, date: ~D[2021-01-01]),
build(:imported_visitors, date: ~D[2021-01-31])
])
conn =
get(
conn,
"/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true"
)
assert %{"plot" => plot, "imported_source" => "Google Analytics"} = json_response(conn, 200)
assert Enum.count(plot) == 31
assert List.first(plot) == 1
assert List.last(plot) == 1
assert Enum.sum(plot) == 2
end
test "displays visitors for a month with imported data and filter", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, timestamp: ~N[2021-01-01 00:00:00], pathname: "/pageA"),
@ -154,6 +193,26 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert Enum.sum(plot) == 4
end
test "displays visitors for 6 months with only imported data", %{conn: conn, site: site} do
populate_stats(site, [
build(:imported_visitors, date: ~D[2021-01-01]),
build(:imported_visitors, date: ~D[2021-06-30])
])
conn =
get(
conn,
"/api/stats/#{site.domain}/main-graph?period=6mo&date=2021-06-30&with_imported=true"
)
assert %{"plot" => plot} = json_response(conn, 200)
assert Enum.count(plot) == 6
assert List.first(plot) == 1
assert List.last(plot) == 1
assert Enum.sum(plot) == 2
end
test "displays visitors for 12 months with imported data", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
@ -176,6 +235,26 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert Enum.sum(plot) == 4
end
test "displays visitors for 12 months with only imported data", %{conn: conn, site: site} do
populate_stats(site, [
build(:imported_visitors, date: ~D[2021-01-01]),
build(:imported_visitors, date: ~D[2021-12-31])
])
conn =
get(
conn,
"/api/stats/#{site.domain}/main-graph?period=12mo&date=2021-12-31&with_imported=true"
)
assert %{"plot" => plot} = json_response(conn, 200)
assert Enum.count(plot) == 12
assert List.first(plot) == 1
assert List.last(plot) == 1
assert Enum.sum(plot) == 2
end
test "displays visitors for calendar year with imported data", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
@ -198,6 +277,26 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert Enum.sum(plot) == 4
end
test "displays visitors for calendar year with only imported data", %{conn: conn, site: site} do
populate_stats(site, [
build(:imported_visitors, date: ~D[2021-01-01]),
build(:imported_visitors, date: ~D[2021-12-31])
])
conn =
get(
conn,
"/api/stats/#{site.domain}/main-graph?period=year&date=2021-12-31&with_imported=true"
)
assert %{"plot" => plot} = json_response(conn, 200)
assert Enum.count(plot) == 12
assert List.first(plot) == 1
assert List.last(plot) == 1
assert Enum.sum(plot) == 2
end
test "displays visitors for all time with just native data", %{conn: conn, site: site} do
use Plausible.Repo
@ -292,6 +391,26 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert List.last(plot) == 2
assert Enum.sum(plot) == 4
end
test "displays pageviews for a month with only imported data", %{conn: conn, site: site} do
populate_stats(site, [
build(:imported_visitors, date: ~D[2021-01-01]),
build(:imported_visitors, date: ~D[2021-01-31])
])
conn =
get(
conn,
"/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=pageviews&with_imported=true"
)
assert %{"plot" => plot} = json_response(conn, 200)
assert Enum.count(plot) == 31
assert List.first(plot) == 1
assert List.last(plot) == 1
assert Enum.sum(plot) == 2
end
end
describe "GET /api/stats/main-graph - bounce_rate plot" do
@ -337,6 +456,25 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert List.first(plot) == 50
assert List.last(plot) == 100
end
test "displays bounce rate for a month with only imported data", %{conn: conn, site: site} do
populate_stats(site, [
build(:imported_visitors, visits: 1, bounces: 0, date: ~D[2021-01-01]),
build(:imported_visitors, visits: 1, bounces: 1, date: ~D[2021-01-31])
])
conn =
get(
conn,
"/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=bounce_rate&with_imported=true"
)
assert %{"plot" => plot} = json_response(conn, 200)
assert Enum.count(plot) == 31
assert List.first(plot) == 0
assert List.last(plot) == 100
end
end
describe "GET /api/stats/main-graph - visit_duration plot" do
@ -387,6 +525,23 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert Enum.count(plot) == 31
assert List.first(plot) == 200
end
test "displays visit_duration for a month with only imported data", %{conn: conn, site: site} do
populate_stats(site, [
build(:imported_visitors, visits: 1, visit_duration: 100, date: ~D[2021-01-01])
])
conn =
get(
conn,
"/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=visit_duration&with_imported=true"
)
assert %{"plot" => plot} = json_response(conn, 200)
assert Enum.count(plot) == 31
assert List.first(plot) == 100
end
end
describe "GET /api/stats/main-graph - varying intervals" do

33
test/support/factory.ex
View File

@ -44,26 +44,12 @@ defmodule Plausible.Factory do
user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
hostname: hostname,
domain: hostname,
referrer: "",
referrer_source: "",
utm_medium: "",
utm_source: "",
utm_campaign: "",
utm_content: "",
utm_term: "",
entry_page: "/",
pageviews: 1,
events: 1,
duration: 0,
start: Timex.now(),
timestamp: Timex.now(),
is_bounce: false,
browser: "",
browser_version: "",
country_code: "",
screen_size: "",
operating_system: "",
operating_system_version: ""
is_bounce: false
}
end
@ -85,22 +71,7 @@ defmodule Plausible.Factory do
pathname: "/",
timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second),
user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
referrer: "",
referrer_source: "",
utm_medium: "",
utm_source: "",
utm_campaign: "",
utm_content: "",
utm_term: "",
browser: "",
browser_version: "",
country_code: "",
screen_size: "",
operating_system: "",
operating_system_version: "",
"meta.key": [],
"meta.value": []
session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate())
}
end

44
test/support/test_utils.ex
View File

@ -73,28 +73,38 @@ defmodule Plausible.TestUtils do
def create_pageviews(pageviews) do
pageviews =
Enum.map(pageviews, fn pageview ->
Factory.build(:pageview, pageview) |> Map.from_struct() |> Map.delete(:__meta__)
Factory.build(:pageview, pageview)
|> Map.from_struct()
|> Map.delete(:__meta__)
|> update_in([:timestamp], &to_naive_truncate/1)
end)
Plausible.IngestRepo.insert_all("events", pageviews)
Plausible.IngestRepo.insert_all(Plausible.ClickhouseEvent, pageviews)
end
def create_events(events) do
events =
Enum.map(events, fn event ->
Factory.build(:event, event) |> Map.from_struct() |> Map.delete(:__meta__)
Factory.build(:event, event)
|> Map.from_struct()
|> Map.delete(:__meta__)
|> update_in([:timestamp], &to_naive_truncate/1)
end)
Plausible.IngestRepo.insert_all("events", events)
Plausible.IngestRepo.insert_all(Plausible.ClickhouseEvent, events)
end
def create_sessions(sessions) do
sessions =
Enum.map(sessions, fn session ->
Factory.build(:ch_session, session) |> Map.from_struct() |> Map.delete(:__meta__)
Factory.build(:ch_session, session)
|> Map.from_struct()
|> Map.delete(:__meta__)
|> update_in([:timestamp], &to_naive_truncate/1)
|> update_in([:start], &to_naive_truncate/1)
end)
Plausible.IngestRepo.insert_all("sessions", sessions)
Plausible.IngestRepo.insert_all(Plausible.ClickhouseSession, sessions)
end
def log_in(%{user: user, conn: conn}) do
@ -136,7 +146,17 @@ defmodule Plausible.TestUtils do
def populate_stats(events) do
{native, imported} =
Enum.split_with(events, fn event ->
events
|> Enum.map(fn event ->
case event do
%{timestamp: timestamp} ->
%{event | timestamp: to_naive_truncate(timestamp)}
_other ->
event
end
end)
|> Enum.split_with(fn event ->
case event do
%Plausible.ClickhouseEvent{} ->
true
@ -168,7 +188,7 @@ defmodule Plausible.TestUtils do
defp populate_imported_stats(events) do
Enum.group_by(events, &Map.fetch!(&1, :table), &Map.delete(&1, :table))
|> Enum.map(fn {table, events} -> Plausible.IngestRepo.insert_all(table, events) end)
|> Enum.map(fn {table, events} -> Plausible.Google.Buffer.insert_all(table, events) end)
end
def relative_time(shifts) do
@ -177,6 +197,14 @@ defmodule Plausible.TestUtils do
|> NaiveDateTime.truncate(:second)
end
def to_naive_truncate(%DateTime{} = dt) do
to_naive_truncate(DateTime.to_naive(dt))
end
def to_naive_truncate(%NaiveDateTime{} = naive) do
NaiveDateTime.truncate(naive, :second)
end
def eventually(expectation, wait_time_ms \\ 50, retries \\ 10) do
Enum.reduce_while(1..retries, nil, fn attempt, _acc ->
case expectation.() do