plausible/analytics
1
1
Fork 0
mirror of https://github.com/plausible/analytics.git synced 2024-12-23 01:22:15 +03:00
Code Issues Packages Projects Releases Wiki Activity

Use Clickhouse everywhere

Browse Source
This commit is contained in:
Uku Taht 2020-05-22 12:33:17 +03:00
parent 439a4a80c7
commit b31c0114c5
20 changed files with 123 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lib/mix/tasks/send_site_setup_emails.ex
View File

@ -2,6 +2,7 @@ defmodule Mix.Tasks.SendSiteSetupEmails do
use Mix.Task
use Plausible.Repo
require Logger
alias Plausible.Stats.Clickhouse, as: Stats
@doc """
This is scheduled to run every 6 hours.
@ -46,7 +47,7 @@ defmodule Mix.Tasks.SendSiteSetupEmails do
for site <- Repo.all(q) do
owner = List.first(site.members)
setup_completed = Plausible.Sites.has_pageviews?(site)
setup_completed = Stats.has_pageviews?(site)
hours_passed = Timex.diff(Timex.now(), site.inserted_at, :hours)
if !setup_completed && hours_passed > 47 do
@ -67,7 +68,7 @@ defmodule Mix.Tasks.SendSiteSetupEmails do
for site <- Repo.all(q) do
owner = List.first(site.members)
if Plausible.Sites.has_pageviews?(site) do
if Stats.has_pageviews?(site) do
send_setup_success_email(args, owner, site)
end
end

24
lib/plausible/auth/auth.ex
View File

@ -1,6 +1,7 @@
defmodule Plausible.Auth do
use Plausible.Repo
alias Plausible.Auth
alias Plausible.Stats.Clickhouse, as: Stats
def create_user(name, email) do
%Auth.User{}
@ -13,18 +14,15 @@ defmodule Plausible.Auth do
end
def user_completed_setup?(user) do
query =
from(
e in Plausible.Event,
join: s in Plausible.Site,
on: s.domain == e.domain,
join: sm in Plausible.Site.Membership,
on: sm.site_id == s.id,
join: u in Plausible.Auth.User,
on: sm.user_id == u.id,
where: u.id == ^user.id
)
Repo.exists?(query)
domains = Repo.all(
from u in Plausible.Auth.User,
where: u.id == ^user.id,
join: sm in Plausible.Site.Membership,
on: sm.user_id == u.id,
join: s in Plausible.Site,
on: s.id == sm.site_id,
select: s.domain
)
Stats.has_pageviews?(domains)
end
end

8
lib/plausible/billing/billing.ex
View File

@ -105,11 +105,9 @@ defmodule Plausible.Billing do
end
defp site_usage(site) do
Repo.aggregate(from(
e in Plausible.Event,
where: e.domain == ^site.domain,
where: e.timestamp >= fragment("now() - '30 days'::interval")
), :count, :id)
q = Plausible.Stats.Query.from(site.timezone, %{"period" => "30d"})
{pageviews, _} = Plausible.Stats.Clickhouse.pageviews_and_visitors(site, q)
pageviews
end
defp format_subscription(params) do

5
lib/plausible/clickhouse.ex
View File

@ -35,9 +35,8 @@ defmodule Plausible.Clickhouse do
Clickhousex.query(:clickhouse, insert, args, log: {Plausible.Clickhouse, :log, []})
end
def escape_quote(s) do
String.replace(s, "'", "''")
end
def escape_quote(nil), do: nil
def escape_quote(s), do: String.replace(s, "'", "''")
def log(query) do
require Logger

7
lib/plausible/sites.ex
View File

@ -12,13 +12,6 @@ defmodule Plausible.Sites do
)
end
def has_pageviews?(site) do
Repo.exists?(
from e in Plausible.Event,
where: e.domain == ^site.domain
)
end
def has_goals?(site) do
Repo.exists?(
from g in Plausible.Goal,

12
lib/plausible/stats/clickhouse.ex
View File

@ -342,7 +342,7 @@ defmodule Plausible.Stats.Clickhouse do
from e in base_query(site, query),
select: {fragment("? as name", e.country_code), fragment("uniq(user_id) as count")},
group_by: e.country_code,
where: e.country_code != "",
where: e.country_code != "\0\0",
order_by: [desc: fragment("count")]
)
|> Enum.map(fn stat ->
@ -389,6 +389,16 @@ defmodule Plausible.Stats.Clickhouse do
res["visitors"]
end
def has_pageviews?(domains) when is_list(domains) do
res = Clickhouse.all(
from e in "events",
select: e.timestamp,
where: fragment("? IN tuple(?)", e.domain, ^domains),
limit: 1
)
!Enum.empty?(res)
end
def has_pageviews?(site) do
res = Clickhouse.all(
from e in "events",

8
lib/plausible_web/controllers/api/internal_controller.ex
View File

@ -1,14 +1,10 @@
defmodule PlausibleWeb.Api.InternalController do
use PlausibleWeb, :controller
use Plausible.Repo
alias Plausible.Stats.Clickhouse, as: Stats
def domain_status(conn, %{"domain" => domain}) do
has_pageviews = Repo.exists?(
from e in Plausible.Event,
where: e.domain == ^domain
)
if has_pageviews do
if Stats.has_pageviews?(%Plausible.Site{domain: domain}) do
json(conn, "READY")
else
json(conn, "WAITING")

1
lib/plausible_web/controllers/site_controller.ex
View File

@ -142,7 +142,6 @@ defmodule PlausibleWeb.SiteController do
|> Repo.preload(:google_auth)
Repo.delete_all(from sm in "site_memberships", where: sm.site_id == ^site.id)
Repo.delete_all(from e in "events", where: e.domain == ^site.domain)
if site.google_auth do
Repo.delete!(site.google_auth)

12
test/mix/tasks/send_check_stats_emails_test.exs
View File

@ -4,8 +4,7 @@ defmodule Mix.Tasks.SendCheckStatsEmailsTest do
test "does not send an email before a week has passed" do
user = insert(:user, inserted_at: days_ago(6), last_seen: days_ago(6))
site = insert(:site, members: [user])
insert(:pageview, domain: site.domain)
insert(:site, domain: "test-site.com", members: [user])
Mix.Tasks.SendCheckStatsEmails.execute()
@ -14,8 +13,7 @@ defmodule Mix.Tasks.SendCheckStatsEmailsTest do
test "does not send an email if the user has logged in recently" do
user = insert(:user, inserted_at: days_ago(9), last_seen: days_ago(6))
site = insert(:site, members: [user])
insert(:pageview, domain: site.domain)
insert(:site, domain: "test-site.com", members: [user])
Mix.Tasks.SendCheckStatsEmails.execute()
@ -24,9 +22,8 @@ defmodule Mix.Tasks.SendCheckStatsEmailsTest do
test "does not send an email if the user has configured a weekly report" do
user = insert(:user, inserted_at: days_ago(9), last_seen: days_ago(7))
site = insert(:site, members: [user])
site = insert(:site, domain: "test-site.com", members: [user])
insert(:weekly_report, site: site, recipients: ["user@email.com"])
insert(:pageview, domain: site.domain)
Mix.Tasks.SendCheckStatsEmails.execute()
@ -35,8 +32,7 @@ defmodule Mix.Tasks.SendCheckStatsEmailsTest do
test "sends an email after a week of signup if the user hasn't logged in" do
user = insert(:user, inserted_at: days_ago(8), last_seen: days_ago(8))
site = insert(:site, members: [user])
insert(:pageview, domain: site.domain)
insert(:site, domain: "test-site.com", members: [user])
Mix.Tasks.SendCheckStatsEmails.execute()

6
test/mix/tasks/send_site_setup_emails_test.exs
View File

@ -1,6 +1,7 @@
defmodule Mix.Tasks.SendSiteSetupEmailsTest do
use Plausible.DataCase
use Bamboo.Test
import Plausible.TestUtils
describe "when user has not managed to set up the site" do
test "does not send an email 47 hours after site creation" do
@ -37,8 +38,7 @@ defmodule Mix.Tasks.SendSiteSetupEmailsTest do
describe "when user has managed to set up their site" do
test "sends the setup completed email as soon as possible" do
user = insert(:user)
site = insert(:site, members: [user])
insert(:pageview, domain: site.domain)
insert(:site, members: [user], domain: "test-site.com")
Mix.Tasks.SendSiteSetupEmails.execute()
@ -59,7 +59,7 @@ defmodule Mix.Tasks.SendSiteSetupEmailsTest do
subject: "Your Plausible setup: Waiting for the first page views"
)
insert(:pageview, domain: site.domain)
create_pageviews([%{domain: site.domain}])
Mix.Tasks.SendSiteSetupEmails.execute()
assert_email_delivered_with(

28
test/mix/tasks/send_trial_notifications_test.exs
View File

@ -16,8 +16,7 @@ defmodule Mix.Tasks.SendTrialNotificationsTest do
describe "with site and pageviews" do
test "sends a reminder 7 days before trial ends (16 days after user signed up)" do
user = insert(:user, trial_expiry_date: Timex.now |> Timex.shift(days: 7))
site = insert(:site, members: [user])
insert(:pageview, domain: site.domain)
insert(:site, domain: "test-site.com", members: [user])
Mix.Tasks.SendTrialNotifications.execute()
@ -26,28 +25,25 @@ defmodule Mix.Tasks.SendTrialNotificationsTest do
test "sends an upgrade email the day before the trial ends" do
user = insert(:user, trial_expiry_date: Timex.now |> Timex.shift(days: 1))
site = insert(:site, members: [user])
insert(:pageview, domain: site.domain)
insert(:site, domain: "test-site.com", members: [user])
Mix.Tasks.SendTrialNotifications.execute()
assert_delivered_email(PlausibleWeb.Email.trial_upgrade_email(user, "tomorrow", 1))
assert_delivered_email(PlausibleWeb.Email.trial_upgrade_email(user, "tomorrow", 3))
end
test "sends an upgrade email the day the trial ends" do
user = insert(:user, trial_expiry_date: Timex.today())
site = insert(:site, members: [user])
insert(:pageview, domain: site.domain)
insert(:site, domain: "test-site.com", members: [user])
Mix.Tasks.SendTrialNotifications.execute()
assert_delivered_email(PlausibleWeb.Email.trial_upgrade_email(user, "today", 1))
assert_delivered_email(PlausibleWeb.Email.trial_upgrade_email(user, "today", 3))
end
test "sends a trial over email the day after the trial ends" do
user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: -1))
site = insert(:site, members: [user])
insert(:pageview, domain: site.domain)
insert(:site, domain: "test-site.com", members: [user])
Mix.Tasks.SendTrialNotifications.execute()
@ -55,15 +51,9 @@ defmodule Mix.Tasks.SendTrialNotificationsTest do
end
test "does not send a notification if user has a subscription" do
user1 = insert(:user, trial_expiry_date: Timex.now |> Timex.shift(days: 7))
site1 = insert(:site, members: [user1])
insert(:pageview, domain: site1.domain)
user2 = insert(:user, trial_expiry_date: Timex.now |> Timex.shift(days: 1))
site2 = insert(:site, members: [user2])
insert(:pageview, domain: site2.domain)
insert(:subscription, user: user1)
insert(:subscription, user: user2)
user = insert(:user, trial_expiry_date: Timex.now |> Timex.shift(days: 7))
insert(:site, domain: "test-site.com", members: [user])
insert(:subscription, user: user)
Mix.Tasks.SendTrialNotifications.execute()

3
test/plausible/auth/auth_test.exs
View File

@ -12,8 +12,7 @@ defmodule Plausible.AuthTest do
test "is true if user does have events" do
user = insert(:user)
site = insert(:site, members: [user])
insert(:pageview, domain: site.domain)
insert(:site, members: [user], domain: "test-site.com")
assert Auth.user_completed_setup?(user)
end

6
test/plausible/billing/billing_test.exs
View File

@ -11,11 +11,9 @@ defmodule Plausible.BillingTest do
test "counts the total number of events" do
user = insert(:user)
site = insert(:site, members: [user])
insert(:pageview, domain: site.domain)
insert(:pageview, domain: site.domain)
insert(:site, domain: "test-site.com", members: [user])
assert Billing.usage(user) == 2
assert Billing.usage(user) == 3
end
end

16
test/plausible/ingest/fingerprint_session_test.exs
View File

@ -13,7 +13,7 @@ defmodule Plausible.Ingest.FingerprintSessionTest do
describe "on_event/1" do
test "starts a new session if there is no session for user id" do
pageview = insert(:pageview)
pageview = insert(:pg_pageview)
refute is_pid(:global.whereis_name(pageview.fingerprint))
@ -23,7 +23,7 @@ defmodule Plausible.Ingest.FingerprintSessionTest do
end
test "copies event data to session" do
pageview = insert(:pageview)
pageview = insert(:pg_pageview)
Ingest.FingerprintSession.on_event(pageview)
@ -34,7 +34,7 @@ defmodule Plausible.Ingest.FingerprintSessionTest do
end
test "inserts bounced session when timeout fires after one pageview" do
pageview = insert(:pageview)
pageview = insert(:pg_pageview)
Ingest.FingerprintSession.on_event(pageview)
@ -43,8 +43,8 @@ defmodule Plausible.Ingest.FingerprintSessionTest do
end
test "session with two events is not a bounce" do
pageview = insert(:pageview)
pageview2 = insert(:pageview, fingerprint: pageview.fingerprint)
pageview = insert(:pg_pageview)
pageview2 = insert(:pg_pageview, fingerprint: pageview.fingerprint)
Ingest.FingerprintSession.on_event(pageview)
Ingest.FingerprintSession.on_event(pageview2)
@ -54,8 +54,8 @@ defmodule Plausible.Ingest.FingerprintSessionTest do
end
test "captures the exit page" do
pageview = insert(:pageview)
pageview2 = insert(:pageview, fingerprint: pageview.fingerprint, pathname: "/exit")
pageview = insert(:pg_pageview)
pageview2 = insert(:pg_pageview, fingerprint: pageview.fingerprint, pathname: "/exit")
Ingest.FingerprintSession.on_event(pageview)
Ingest.FingerprintSession.on_event(pageview2)
@ -67,7 +67,7 @@ defmodule Plausible.Ingest.FingerprintSessionTest do
describe "on_unload/1" do
test "uses the unload timestamp to calculate session length" do
pageview = insert(:pageview)
pageview = insert(:pg_pageview)
unload_timestamp = Timex.shift(pageview.timestamp, seconds: 30)
Ingest.FingerprintSession.on_event(pageview)

15
test/plausible/sites_test.exs
View File

@ -2,21 +2,6 @@ defmodule Plausible.SitesTest do
use Plausible.DataCase
alias Plausible.Sites
describe "has_pageviews?" do
test "is true if site has pageviews" do
site = insert(:site)
insert(:pageview, domain: site.domain)
assert Sites.has_pageviews?(site)
end
test "is false if site does not have pageviews" do
site = insert(:site)
refute Sites.has_pageviews?(site)
end
end
describe "is_owner?" do
test "is true if user is the owner of the site" do
user = insert(:user)

10
test/plausible_web/controllers/api/internal_controller_test.exs
View File

@ -4,16 +4,18 @@ defmodule PlausibleWeb.Api.InternalControllerTest do
import Plausible.TestUtils
describe "GET /:domain/status" do
setup [:create_user, :log_in, :create_site]
setup [:create_user, :log_in]
test "is WAITING when site has no pageviews", %{conn: conn, site: site} do
test "is WAITING when site has no pageviews", %{conn: conn, user: user} do
site = insert(:site, members: [user])
conn = get(conn, "/api/#{site.domain}/status")
assert json_response(conn, 200) == "WAITING"
end
test "is READY when site has at least 1 pageview", %{conn: conn, site: site} do
insert(:pageview, domain: site.domain)
test "is READY when site has at least 1 pageview", %{conn: conn, user: user} do
site = insert(:site, members: [user])
Plausible.TestUtils.create_pageviews([%{domain: site.domain}])
conn = get(conn, "/api/#{site.domain}/status")

16
test/plausible_web/controllers/api/stats_controller/referrers_test.exs
View File

@ -9,8 +9,8 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do
conn = get(conn, "/api/stats/#{site.domain}/referrers?period=day&date=2019-01-01")
assert json_response(conn, 200) == [
%{"name" => "10words", "count" => 2, "url" => nil},
%{"name" => "Bing", "count" => 1, "url" => nil},
%{"name" => "10words", "count" => 2, "url" => "10words.com"},
%{"name" => "Bing", "count" => 1, "url" => ""},
]
end
@ -18,8 +18,8 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do
conn = get(conn, "/api/stats/#{site.domain}/referrers?period=day&date=2019-01-01&include=bounce_rate")
assert json_response(conn, 200) == [
%{"name" => "10words", "count" => 2, "bounce_rate" => 50, "url" => nil},
%{"name" => "Bing", "count" => 1, "bounce_rate" => nil, "url" => nil},
%{"name" => "10words", "count" => 2, "bounce_rate" => 50.0, "url" => "10words.com"},
%{"name" => "Bing", "count" => 1, "bounce_rate" => nil, "url" => ""},
]
end
end
@ -32,7 +32,7 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do
conn = get(conn, "/api/stats/#{site.domain}/goal/referrers?period=day&date=2019-01-01&filters=#{filters}")
assert json_response(conn, 200) == [
%{"name" => "Google", "count" => 3, "url" => nil},
%{"name" => "Google", "count" => 3, "url" => "google.com"},
]
end
@ -41,7 +41,7 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do
conn = get(conn, "/api/stats/#{site.domain}/goal/referrers?period=day&date=2019-01-01&filters=#{filters}")
assert json_response(conn, 200) == [
%{"name" => "Google", "count" => 2, "url" => nil},
%{"name" => "Google", "count" => 2, "url" => "google.com"},
]
end
end
@ -55,8 +55,8 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do
assert json_response(conn, 200) == %{
"total_visitors" => 2,
"referrers" => [
%{"name" => "10words.com/page1", "count" => 1},
%{"name" => "10words.com/page2", "count" => 1},
%{"name" => "10words.com/page1", "count" => 1},
]
}
end
@ -67,8 +67,8 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do
assert json_response(conn, 200) == %{
"total_visitors" => 2,
"referrers" => [
%{"name" => "10words.com/page1", "count" => 1, "bounce_rate" => 50},
%{"name" => "10words.com/page2", "count" => 1, "bounce_rate" => nil},
%{"name" => "10words.com/page1", "count" => 1, "bounce_rate" => 50.0},
]
}
end

4
test/plausible_web/controllers/site_controller_test.exs
View File

@ -133,15 +133,13 @@ defmodule PlausibleWeb.SiteControllerTest do
describe "DELETE /:website" do
setup [:create_user, :log_in, :create_site]
test "deletes the site and all pageviews", %{conn: conn, user: user, site: site} do
pageview = insert(:pageview, domain: site.domain)
test "deletes the site", %{conn: conn, user: user, site: site} do
insert(:google_auth, user: user, site: site)
insert(:custom_domain, site: site)
delete(conn, "/#{site.domain}")
refute Repo.exists?(from s in Plausible.Site, where: s.id == ^site.id)
refute Repo.exists?(from e in Plausible.Event, where: e.id == ^pageview.id)
end
end

47
test/support/clickhouse_setup.ex
View File

@ -12,20 +12,21 @@ defmodule Plausible.Test.ClickhouseSetup do
timestamp DateTime,
name String,
domain String,
user_id FixedString(64),
user_id UInt64,
session_id UInt64,
hostname String,
pathname String,
referrer Nullable(String),
referrer_source Nullable(String),
initial_referrer Nullable(String),
initial_referrer_source Nullable(String),
country_code Nullable(FixedString(2)),
screen_size Nullable(String),
operating_system Nullable(String),
browser Nullable(String)
referrer String,
referrer_source String,
initial_referrer String,
initial_referrer_source String,
country_code LowCardinality(FixedString(2)),
screen_size LowCardinality(String),
operating_system LowCardinality(String),
browser LowCardinality(String)
) ENGINE = MergeTree()
PARTITION BY toYYYYMM(timestamp)
ORDER BY (name, domain, timestamp, user_id)
ORDER BY (name, domain, user_id, timestamp)
SETTINGS index_granularity = 8192
"""
@ -37,28 +38,28 @@ defmodule Plausible.Test.ClickhouseSetup do
drop = "DROP TABLE sessions"
create = """
CREATE TABLE sessions (
session_id UUID,
session_id UInt64,
sign Int8,
domain String,
user_id FixedString(64),
user_id UInt64,
hostname String,
timestamp DateTime,
start DateTime,
is_bounce UInt8,
entry_page Nullable(String),
exit_page Nullable(String),
entry_page String,
exit_page String,
pageviews Int32,
events Int32,
duration UInt32,
referrer Nullable(String),
referrer_source Nullable(String),
country_code Nullable(FixedString(2)),
screen_size Nullable(String),
operating_system Nullable(String),
browser Nullable(String)
referrer String,
referrer_source String,
country_code LowCardinality(FixedString(2)),
screen_size LowCardinality(String),
operating_system LowCardinality(String),
browser LowCardinality(String)
) ENGINE = CollapsingMergeTree(sign)
PARTITION BY toYYYYMM(start)
ORDER BY (domain, start, user_id)
ORDER BY (domain, user_id, session_id, start)
SETTINGS index_granularity = 8192
"""
@ -98,8 +99,8 @@ defmodule Plausible.Test.ClickhouseSetup do
])
Plausible.TestUtils.create_sessions([
%{domain: "test-site.com", entry_page: "/", referrer_source: "10words", referrer: "10words.com/page1", is_bounce: true, start: ~N[2019-01-01 02:00:00]},
%{domain: "test-site.com", entry_page: "/", referrer_source: "10words", referrer: "10words.com/page1", is_bounce: false, start: ~N[2019-01-01 02:00:00]}
%{domain: "test-site.com", entry_page: "/", exit_page: "/", referrer_source: "10words", referrer: "10words.com/page1", is_bounce: true, start: ~N[2019-01-01 02:00:00]},
%{domain: "test-site.com", entry_page: "/", exit_page: "/", referrer_source: "10words", referrer: "10words.com/page1", is_bounce: false, start: ~N[2019-01-01 02:00:00]}
])
end
end

28
test/support/factory.ex
View File

@ -29,14 +29,14 @@ defmodule Plausible.Factory do
%Plausible.ClickhouseSession{
sign: 1,
session_id: UUID.uuid4(),
session_id: SipHash.hash!(@hash_key, UUID.uuid4()),
user_id: SipHash.hash!(@hash_key, UUID.uuid4()),
hostname: hostname,
domain: hostname,
entry_page: "/",
pageviews: 1,
events: 1,
duration: 0,
user_id: UUID.uuid4(),
start: Timex.now(),
timestamp: Timex.now(),
is_bounce: false
@ -56,6 +56,27 @@ defmodule Plausible.Factory do
}
end
def pg_pageview_factory do
struct!(
pg_event_factory(),
%{
name: "pageview"
}
)
end
def pg_event_factory do
hostname = sequence(:domain, &"example-#{&1}.com")
%Plausible.Event{
hostname: hostname,
domain: hostname,
pathname: "/",
timestamp: Timex.now(),
fingerprint: UUID.uuid4()
}
end
def pageview_factory do
struct!(
event_factory(),
@ -73,7 +94,8 @@ defmodule Plausible.Factory do
domain: hostname,
pathname: "/",
timestamp: Timex.now(),
user_id: SipHash.hash!(@hash_key, UUID.uuid4())
user_id: SipHash.hash!(@hash_key, UUID.uuid4()),
session_id: SipHash.hash!(@hash_key, UUID.uuid4())
}
end