diff --git a/lib/mix/tasks/send_site_setup_emails.ex b/lib/mix/tasks/send_site_setup_emails.ex index b1cbe0b87..6b855c6d0 100644 --- a/lib/mix/tasks/send_site_setup_emails.ex +++ b/lib/mix/tasks/send_site_setup_emails.ex @@ -2,6 +2,7 @@ defmodule Mix.Tasks.SendSiteSetupEmails do use Mix.Task use Plausible.Repo require Logger + alias Plausible.Stats.Clickhouse, as: Stats @doc """ This is scheduled to run every 6 hours. @@ -46,7 +47,7 @@ defmodule Mix.Tasks.SendSiteSetupEmails do for site <- Repo.all(q) do owner = List.first(site.members) - setup_completed = Plausible.Sites.has_pageviews?(site) + setup_completed = Stats.has_pageviews?(site) hours_passed = Timex.diff(Timex.now(), site.inserted_at, :hours) if !setup_completed && hours_passed > 47 do @@ -67,7 +68,7 @@ defmodule Mix.Tasks.SendSiteSetupEmails do for site <- Repo.all(q) do owner = List.first(site.members) - if Plausible.Sites.has_pageviews?(site) do + if Stats.has_pageviews?(site) do send_setup_success_email(args, owner, site) end end diff --git a/lib/plausible/auth/auth.ex b/lib/plausible/auth/auth.ex index 3caf0a861..e60e2439f 100644 --- a/lib/plausible/auth/auth.ex +++ b/lib/plausible/auth/auth.ex @@ -1,6 +1,7 @@ defmodule Plausible.Auth do use Plausible.Repo alias Plausible.Auth + alias Plausible.Stats.Clickhouse, as: Stats def create_user(name, email) do %Auth.User{} @@ -13,18 +14,15 @@ defmodule Plausible.Auth do end def user_completed_setup?(user) do - query = - from( - e in Plausible.Event, - join: s in Plausible.Site, - on: s.domain == e.domain, - join: sm in Plausible.Site.Membership, - on: sm.site_id == s.id, - join: u in Plausible.Auth.User, - on: sm.user_id == u.id, - where: u.id == ^user.id - ) - - Repo.exists?(query) + domains = Repo.all( + from u in Plausible.Auth.User, + where: u.id == ^user.id, + join: sm in Plausible.Site.Membership, + on: sm.user_id == u.id, + join: s in Plausible.Site, + on: s.id == sm.site_id, + select: s.domain + ) + Stats.has_pageviews?(domains) end end diff --git a/lib/plausible/billing/billing.ex b/lib/plausible/billing/billing.ex index b9c01cf70..b1c39f4cc 100644 --- a/lib/plausible/billing/billing.ex +++ b/lib/plausible/billing/billing.ex @@ -105,11 +105,9 @@ defmodule Plausible.Billing do end defp site_usage(site) do - Repo.aggregate(from( - e in Plausible.Event, - where: e.domain == ^site.domain, - where: e.timestamp >= fragment("now() - '30 days'::interval") - ), :count, :id) + q = Plausible.Stats.Query.from(site.timezone, %{"period" => "30d"}) + {pageviews, _} = Plausible.Stats.Clickhouse.pageviews_and_visitors(site, q) + pageviews end defp format_subscription(params) do diff --git a/lib/plausible/clickhouse.ex b/lib/plausible/clickhouse.ex index 9c67b63f7..f41e1396a 100644 --- a/lib/plausible/clickhouse.ex +++ b/lib/plausible/clickhouse.ex @@ -35,9 +35,8 @@ defmodule Plausible.Clickhouse do Clickhousex.query(:clickhouse, insert, args, log: {Plausible.Clickhouse, :log, []}) end - def escape_quote(s) do - String.replace(s, "'", "''") - end + def escape_quote(nil), do: nil + def escape_quote(s), do: String.replace(s, "'", "''") def log(query) do require Logger diff --git a/lib/plausible/sites.ex b/lib/plausible/sites.ex index 9a626e7ef..22470428b 100644 --- a/lib/plausible/sites.ex +++ b/lib/plausible/sites.ex @@ -12,13 +12,6 @@ defmodule Plausible.Sites do ) end - def has_pageviews?(site) do - Repo.exists?( - from e in Plausible.Event, - where: e.domain == ^site.domain - ) - end - def has_goals?(site) do Repo.exists?( from g in Plausible.Goal, diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex index 347ef4357..75a49f7fa 100644 --- a/lib/plausible/stats/clickhouse.ex +++ b/lib/plausible/stats/clickhouse.ex @@ -342,7 +342,7 @@ defmodule Plausible.Stats.Clickhouse do from e in base_query(site, query), select: {fragment("? as name", e.country_code), fragment("uniq(user_id) as count")}, group_by: e.country_code, - where: e.country_code != "", + where: e.country_code != "\0\0", order_by: [desc: fragment("count")] ) |> Enum.map(fn stat -> @@ -389,6 +389,16 @@ defmodule Plausible.Stats.Clickhouse do res["visitors"] end + def has_pageviews?(domains) when is_list(domains) do + res = Clickhouse.all( + from e in "events", + select: e.timestamp, + where: fragment("? IN tuple(?)", e.domain, ^domains), + limit: 1 + ) + !Enum.empty?(res) + end + def has_pageviews?(site) do res = Clickhouse.all( from e in "events", diff --git a/lib/plausible_web/controllers/api/internal_controller.ex b/lib/plausible_web/controllers/api/internal_controller.ex index f4e3785f3..6cefc340c 100644 --- a/lib/plausible_web/controllers/api/internal_controller.ex +++ b/lib/plausible_web/controllers/api/internal_controller.ex @@ -1,14 +1,10 @@ defmodule PlausibleWeb.Api.InternalController do use PlausibleWeb, :controller use Plausible.Repo + alias Plausible.Stats.Clickhouse, as: Stats def domain_status(conn, %{"domain" => domain}) do - has_pageviews = Repo.exists?( - from e in Plausible.Event, - where: e.domain == ^domain - ) - - if has_pageviews do + if Stats.has_pageviews?(%Plausible.Site{domain: domain}) do json(conn, "READY") else json(conn, "WAITING") diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 4172b564f..cceb564f1 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -142,7 +142,6 @@ defmodule PlausibleWeb.SiteController do |> Repo.preload(:google_auth) Repo.delete_all(from sm in "site_memberships", where: sm.site_id == ^site.id) - Repo.delete_all(from e in "events", where: e.domain == ^site.domain) if site.google_auth do Repo.delete!(site.google_auth) diff --git a/test/mix/tasks/send_check_stats_emails_test.exs b/test/mix/tasks/send_check_stats_emails_test.exs index 9a3e5ed16..0f6bbde3f 100644 --- a/test/mix/tasks/send_check_stats_emails_test.exs +++ b/test/mix/tasks/send_check_stats_emails_test.exs @@ -4,8 +4,7 @@ defmodule Mix.Tasks.SendCheckStatsEmailsTest do test "does not send an email before a week has passed" do user = insert(:user, inserted_at: days_ago(6), last_seen: days_ago(6)) - site = insert(:site, members: [user]) - insert(:pageview, domain: site.domain) + insert(:site, domain: "test-site.com", members: [user]) Mix.Tasks.SendCheckStatsEmails.execute() @@ -14,8 +13,7 @@ defmodule Mix.Tasks.SendCheckStatsEmailsTest do test "does not send an email if the user has logged in recently" do user = insert(:user, inserted_at: days_ago(9), last_seen: days_ago(6)) - site = insert(:site, members: [user]) - insert(:pageview, domain: site.domain) + insert(:site, domain: "test-site.com", members: [user]) Mix.Tasks.SendCheckStatsEmails.execute() @@ -24,9 +22,8 @@ defmodule Mix.Tasks.SendCheckStatsEmailsTest do test "does not send an email if the user has configured a weekly report" do user = insert(:user, inserted_at: days_ago(9), last_seen: days_ago(7)) - site = insert(:site, members: [user]) + site = insert(:site, domain: "test-site.com", members: [user]) insert(:weekly_report, site: site, recipients: ["user@email.com"]) - insert(:pageview, domain: site.domain) Mix.Tasks.SendCheckStatsEmails.execute() @@ -35,8 +32,7 @@ defmodule Mix.Tasks.SendCheckStatsEmailsTest do test "sends an email after a week of signup if the user hasn't logged in" do user = insert(:user, inserted_at: days_ago(8), last_seen: days_ago(8)) - site = insert(:site, members: [user]) - insert(:pageview, domain: site.domain) + insert(:site, domain: "test-site.com", members: [user]) Mix.Tasks.SendCheckStatsEmails.execute() diff --git a/test/mix/tasks/send_site_setup_emails_test.exs b/test/mix/tasks/send_site_setup_emails_test.exs index 0bf0de7b1..7f0fb1c54 100644 --- a/test/mix/tasks/send_site_setup_emails_test.exs +++ b/test/mix/tasks/send_site_setup_emails_test.exs @@ -1,6 +1,7 @@ defmodule Mix.Tasks.SendSiteSetupEmailsTest do use Plausible.DataCase use Bamboo.Test + import Plausible.TestUtils describe "when user has not managed to set up the site" do test "does not send an email 47 hours after site creation" do @@ -37,8 +38,7 @@ defmodule Mix.Tasks.SendSiteSetupEmailsTest do describe "when user has managed to set up their site" do test "sends the setup completed email as soon as possible" do user = insert(:user) - site = insert(:site, members: [user]) - insert(:pageview, domain: site.domain) + insert(:site, members: [user], domain: "test-site.com") Mix.Tasks.SendSiteSetupEmails.execute() @@ -59,7 +59,7 @@ defmodule Mix.Tasks.SendSiteSetupEmailsTest do subject: "Your Plausible setup: Waiting for the first page views" ) - insert(:pageview, domain: site.domain) + create_pageviews([%{domain: site.domain}]) Mix.Tasks.SendSiteSetupEmails.execute() assert_email_delivered_with( diff --git a/test/mix/tasks/send_trial_notifications_test.exs b/test/mix/tasks/send_trial_notifications_test.exs index a72339699..9787dbe38 100644 --- a/test/mix/tasks/send_trial_notifications_test.exs +++ b/test/mix/tasks/send_trial_notifications_test.exs @@ -16,8 +16,7 @@ defmodule Mix.Tasks.SendTrialNotificationsTest do describe "with site and pageviews" do test "sends a reminder 7 days before trial ends (16 days after user signed up)" do user = insert(:user, trial_expiry_date: Timex.now |> Timex.shift(days: 7)) - site = insert(:site, members: [user]) - insert(:pageview, domain: site.domain) + insert(:site, domain: "test-site.com", members: [user]) Mix.Tasks.SendTrialNotifications.execute() @@ -26,28 +25,25 @@ defmodule Mix.Tasks.SendTrialNotificationsTest do test "sends an upgrade email the day before the trial ends" do user = insert(:user, trial_expiry_date: Timex.now |> Timex.shift(days: 1)) - site = insert(:site, members: [user]) - insert(:pageview, domain: site.domain) + insert(:site, domain: "test-site.com", members: [user]) Mix.Tasks.SendTrialNotifications.execute() - assert_delivered_email(PlausibleWeb.Email.trial_upgrade_email(user, "tomorrow", 1)) + assert_delivered_email(PlausibleWeb.Email.trial_upgrade_email(user, "tomorrow", 3)) end test "sends an upgrade email the day the trial ends" do user = insert(:user, trial_expiry_date: Timex.today()) - site = insert(:site, members: [user]) - insert(:pageview, domain: site.domain) + insert(:site, domain: "test-site.com", members: [user]) Mix.Tasks.SendTrialNotifications.execute() - assert_delivered_email(PlausibleWeb.Email.trial_upgrade_email(user, "today", 1)) + assert_delivered_email(PlausibleWeb.Email.trial_upgrade_email(user, "today", 3)) end test "sends a trial over email the day after the trial ends" do user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: -1)) - site = insert(:site, members: [user]) - insert(:pageview, domain: site.domain) + insert(:site, domain: "test-site.com", members: [user]) Mix.Tasks.SendTrialNotifications.execute() @@ -55,15 +51,9 @@ defmodule Mix.Tasks.SendTrialNotificationsTest do end test "does not send a notification if user has a subscription" do - user1 = insert(:user, trial_expiry_date: Timex.now |> Timex.shift(days: 7)) - site1 = insert(:site, members: [user1]) - insert(:pageview, domain: site1.domain) - user2 = insert(:user, trial_expiry_date: Timex.now |> Timex.shift(days: 1)) - site2 = insert(:site, members: [user2]) - insert(:pageview, domain: site2.domain) - - insert(:subscription, user: user1) - insert(:subscription, user: user2) + user = insert(:user, trial_expiry_date: Timex.now |> Timex.shift(days: 7)) + insert(:site, domain: "test-site.com", members: [user]) + insert(:subscription, user: user) Mix.Tasks.SendTrialNotifications.execute() diff --git a/test/plausible/auth/auth_test.exs b/test/plausible/auth/auth_test.exs index 4321fb828..3b792face 100644 --- a/test/plausible/auth/auth_test.exs +++ b/test/plausible/auth/auth_test.exs @@ -12,8 +12,7 @@ defmodule Plausible.AuthTest do test "is true if user does have events" do user = insert(:user) - site = insert(:site, members: [user]) - insert(:pageview, domain: site.domain) + insert(:site, members: [user], domain: "test-site.com") assert Auth.user_completed_setup?(user) end diff --git a/test/plausible/billing/billing_test.exs b/test/plausible/billing/billing_test.exs index b990dff3b..41074a52c 100644 --- a/test/plausible/billing/billing_test.exs +++ b/test/plausible/billing/billing_test.exs @@ -11,11 +11,9 @@ defmodule Plausible.BillingTest do test "counts the total number of events" do user = insert(:user) - site = insert(:site, members: [user]) - insert(:pageview, domain: site.domain) - insert(:pageview, domain: site.domain) + insert(:site, domain: "test-site.com", members: [user]) - assert Billing.usage(user) == 2 + assert Billing.usage(user) == 3 end end diff --git a/test/plausible/ingest/fingerprint_session_test.exs b/test/plausible/ingest/fingerprint_session_test.exs index 2421034ee..9151ea6a7 100644 --- a/test/plausible/ingest/fingerprint_session_test.exs +++ b/test/plausible/ingest/fingerprint_session_test.exs @@ -13,7 +13,7 @@ defmodule Plausible.Ingest.FingerprintSessionTest do describe "on_event/1" do test "starts a new session if there is no session for user id" do - pageview = insert(:pageview) + pageview = insert(:pg_pageview) refute is_pid(:global.whereis_name(pageview.fingerprint)) @@ -23,7 +23,7 @@ defmodule Plausible.Ingest.FingerprintSessionTest do end test "copies event data to session" do - pageview = insert(:pageview) + pageview = insert(:pg_pageview) Ingest.FingerprintSession.on_event(pageview) @@ -34,7 +34,7 @@ defmodule Plausible.Ingest.FingerprintSessionTest do end test "inserts bounced session when timeout fires after one pageview" do - pageview = insert(:pageview) + pageview = insert(:pg_pageview) Ingest.FingerprintSession.on_event(pageview) @@ -43,8 +43,8 @@ defmodule Plausible.Ingest.FingerprintSessionTest do end test "session with two events is not a bounce" do - pageview = insert(:pageview) - pageview2 = insert(:pageview, fingerprint: pageview.fingerprint) + pageview = insert(:pg_pageview) + pageview2 = insert(:pg_pageview, fingerprint: pageview.fingerprint) Ingest.FingerprintSession.on_event(pageview) Ingest.FingerprintSession.on_event(pageview2) @@ -54,8 +54,8 @@ defmodule Plausible.Ingest.FingerprintSessionTest do end test "captures the exit page" do - pageview = insert(:pageview) - pageview2 = insert(:pageview, fingerprint: pageview.fingerprint, pathname: "/exit") + pageview = insert(:pg_pageview) + pageview2 = insert(:pg_pageview, fingerprint: pageview.fingerprint, pathname: "/exit") Ingest.FingerprintSession.on_event(pageview) Ingest.FingerprintSession.on_event(pageview2) @@ -67,7 +67,7 @@ defmodule Plausible.Ingest.FingerprintSessionTest do describe "on_unload/1" do test "uses the unload timestamp to calculate session length" do - pageview = insert(:pageview) + pageview = insert(:pg_pageview) unload_timestamp = Timex.shift(pageview.timestamp, seconds: 30) Ingest.FingerprintSession.on_event(pageview) diff --git a/test/plausible/sites_test.exs b/test/plausible/sites_test.exs index b1d5655ca..b95a0e1fe 100644 --- a/test/plausible/sites_test.exs +++ b/test/plausible/sites_test.exs @@ -2,21 +2,6 @@ defmodule Plausible.SitesTest do use Plausible.DataCase alias Plausible.Sites - describe "has_pageviews?" do - test "is true if site has pageviews" do - site = insert(:site) - insert(:pageview, domain: site.domain) - - assert Sites.has_pageviews?(site) - end - - test "is false if site does not have pageviews" do - site = insert(:site) - - refute Sites.has_pageviews?(site) - end - end - describe "is_owner?" do test "is true if user is the owner of the site" do user = insert(:user) diff --git a/test/plausible_web/controllers/api/internal_controller_test.exs b/test/plausible_web/controllers/api/internal_controller_test.exs index 6449955f1..9c335191d 100644 --- a/test/plausible_web/controllers/api/internal_controller_test.exs +++ b/test/plausible_web/controllers/api/internal_controller_test.exs @@ -4,16 +4,18 @@ defmodule PlausibleWeb.Api.InternalControllerTest do import Plausible.TestUtils describe "GET /:domain/status" do - setup [:create_user, :log_in, :create_site] + setup [:create_user, :log_in] - test "is WAITING when site has no pageviews", %{conn: conn, site: site} do + test "is WAITING when site has no pageviews", %{conn: conn, user: user} do + site = insert(:site, members: [user]) conn = get(conn, "/api/#{site.domain}/status") assert json_response(conn, 200) == "WAITING" end - test "is READY when site has at least 1 pageview", %{conn: conn, site: site} do - insert(:pageview, domain: site.domain) + test "is READY when site has at least 1 pageview", %{conn: conn, user: user} do + site = insert(:site, members: [user]) + Plausible.TestUtils.create_pageviews([%{domain: site.domain}]) conn = get(conn, "/api/#{site.domain}/status") diff --git a/test/plausible_web/controllers/api/stats_controller/referrers_test.exs b/test/plausible_web/controllers/api/stats_controller/referrers_test.exs index 0c26415ce..d35dfe6a0 100644 --- a/test/plausible_web/controllers/api/stats_controller/referrers_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/referrers_test.exs @@ -9,8 +9,8 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do conn = get(conn, "/api/stats/#{site.domain}/referrers?period=day&date=2019-01-01") assert json_response(conn, 200) == [ - %{"name" => "10words", "count" => 2, "url" => nil}, - %{"name" => "Bing", "count" => 1, "url" => nil}, + %{"name" => "10words", "count" => 2, "url" => "10words.com"}, + %{"name" => "Bing", "count" => 1, "url" => ""}, ] end @@ -18,8 +18,8 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do conn = get(conn, "/api/stats/#{site.domain}/referrers?period=day&date=2019-01-01&include=bounce_rate") assert json_response(conn, 200) == [ - %{"name" => "10words", "count" => 2, "bounce_rate" => 50, "url" => nil}, - %{"name" => "Bing", "count" => 1, "bounce_rate" => nil, "url" => nil}, + %{"name" => "10words", "count" => 2, "bounce_rate" => 50.0, "url" => "10words.com"}, + %{"name" => "Bing", "count" => 1, "bounce_rate" => nil, "url" => ""}, ] end end @@ -32,7 +32,7 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do conn = get(conn, "/api/stats/#{site.domain}/goal/referrers?period=day&date=2019-01-01&filters=#{filters}") assert json_response(conn, 200) == [ - %{"name" => "Google", "count" => 3, "url" => nil}, + %{"name" => "Google", "count" => 3, "url" => "google.com"}, ] end @@ -41,7 +41,7 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do conn = get(conn, "/api/stats/#{site.domain}/goal/referrers?period=day&date=2019-01-01&filters=#{filters}") assert json_response(conn, 200) == [ - %{"name" => "Google", "count" => 2, "url" => nil}, + %{"name" => "Google", "count" => 2, "url" => "google.com"}, ] end end @@ -55,8 +55,8 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do assert json_response(conn, 200) == %{ "total_visitors" => 2, "referrers" => [ - %{"name" => "10words.com/page1", "count" => 1}, %{"name" => "10words.com/page2", "count" => 1}, + %{"name" => "10words.com/page1", "count" => 1}, ] } end @@ -67,8 +67,8 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do assert json_response(conn, 200) == %{ "total_visitors" => 2, "referrers" => [ - %{"name" => "10words.com/page1", "count" => 1, "bounce_rate" => 50}, %{"name" => "10words.com/page2", "count" => 1, "bounce_rate" => nil}, + %{"name" => "10words.com/page1", "count" => 1, "bounce_rate" => 50.0}, ] } end diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exs index e01ac63b9..a20d55a4e 100644 --- a/test/plausible_web/controllers/site_controller_test.exs +++ b/test/plausible_web/controllers/site_controller_test.exs @@ -133,15 +133,13 @@ defmodule PlausibleWeb.SiteControllerTest do describe "DELETE /:website" do setup [:create_user, :log_in, :create_site] - test "deletes the site and all pageviews", %{conn: conn, user: user, site: site} do - pageview = insert(:pageview, domain: site.domain) + test "deletes the site", %{conn: conn, user: user, site: site} do insert(:google_auth, user: user, site: site) insert(:custom_domain, site: site) delete(conn, "/#{site.domain}") refute Repo.exists?(from s in Plausible.Site, where: s.id == ^site.id) - refute Repo.exists?(from e in Plausible.Event, where: e.id == ^pageview.id) end end diff --git a/test/support/clickhouse_setup.ex b/test/support/clickhouse_setup.ex index 304b9b96f..d911a50cd 100644 --- a/test/support/clickhouse_setup.ex +++ b/test/support/clickhouse_setup.ex @@ -12,20 +12,21 @@ defmodule Plausible.Test.ClickhouseSetup do timestamp DateTime, name String, domain String, - user_id FixedString(64), + user_id UInt64, + session_id UInt64, hostname String, pathname String, - referrer Nullable(String), - referrer_source Nullable(String), - initial_referrer Nullable(String), - initial_referrer_source Nullable(String), - country_code Nullable(FixedString(2)), - screen_size Nullable(String), - operating_system Nullable(String), - browser Nullable(String) + referrer String, + referrer_source String, + initial_referrer String, + initial_referrer_source String, + country_code LowCardinality(FixedString(2)), + screen_size LowCardinality(String), + operating_system LowCardinality(String), + browser LowCardinality(String) ) ENGINE = MergeTree() PARTITION BY toYYYYMM(timestamp) - ORDER BY (name, domain, timestamp, user_id) + ORDER BY (name, domain, user_id, timestamp) SETTINGS index_granularity = 8192 """ @@ -37,28 +38,28 @@ defmodule Plausible.Test.ClickhouseSetup do drop = "DROP TABLE sessions" create = """ CREATE TABLE sessions ( - session_id UUID, + session_id UInt64, sign Int8, domain String, - user_id FixedString(64), + user_id UInt64, hostname String, timestamp DateTime, start DateTime, is_bounce UInt8, - entry_page Nullable(String), - exit_page Nullable(String), + entry_page String, + exit_page String, pageviews Int32, events Int32, duration UInt32, - referrer Nullable(String), - referrer_source Nullable(String), - country_code Nullable(FixedString(2)), - screen_size Nullable(String), - operating_system Nullable(String), - browser Nullable(String) + referrer String, + referrer_source String, + country_code LowCardinality(FixedString(2)), + screen_size LowCardinality(String), + operating_system LowCardinality(String), + browser LowCardinality(String) ) ENGINE = CollapsingMergeTree(sign) PARTITION BY toYYYYMM(start) - ORDER BY (domain, start, user_id) + ORDER BY (domain, user_id, session_id, start) SETTINGS index_granularity = 8192 """ @@ -98,8 +99,8 @@ defmodule Plausible.Test.ClickhouseSetup do ]) Plausible.TestUtils.create_sessions([ - %{domain: "test-site.com", entry_page: "/", referrer_source: "10words", referrer: "10words.com/page1", is_bounce: true, start: ~N[2019-01-01 02:00:00]}, - %{domain: "test-site.com", entry_page: "/", referrer_source: "10words", referrer: "10words.com/page1", is_bounce: false, start: ~N[2019-01-01 02:00:00]} + %{domain: "test-site.com", entry_page: "/", exit_page: "/", referrer_source: "10words", referrer: "10words.com/page1", is_bounce: true, start: ~N[2019-01-01 02:00:00]}, + %{domain: "test-site.com", entry_page: "/", exit_page: "/", referrer_source: "10words", referrer: "10words.com/page1", is_bounce: false, start: ~N[2019-01-01 02:00:00]} ]) end end diff --git a/test/support/factory.ex b/test/support/factory.ex index 06fa68b61..0aceee8a4 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -29,14 +29,14 @@ defmodule Plausible.Factory do %Plausible.ClickhouseSession{ sign: 1, - session_id: UUID.uuid4(), + session_id: SipHash.hash!(@hash_key, UUID.uuid4()), + user_id: SipHash.hash!(@hash_key, UUID.uuid4()), hostname: hostname, domain: hostname, entry_page: "/", pageviews: 1, events: 1, duration: 0, - user_id: UUID.uuid4(), start: Timex.now(), timestamp: Timex.now(), is_bounce: false @@ -56,6 +56,27 @@ defmodule Plausible.Factory do } end + def pg_pageview_factory do + struct!( + pg_event_factory(), + %{ + name: "pageview" + } + ) + end + + def pg_event_factory do + hostname = sequence(:domain, &"example-#{&1}.com") + + %Plausible.Event{ + hostname: hostname, + domain: hostname, + pathname: "/", + timestamp: Timex.now(), + fingerprint: UUID.uuid4() + } + end + def pageview_factory do struct!( event_factory(), @@ -73,7 +94,8 @@ defmodule Plausible.Factory do domain: hostname, pathname: "/", timestamp: Timex.now(), - user_id: SipHash.hash!(@hash_key, UUID.uuid4()) + user_id: SipHash.hash!(@hash_key, UUID.uuid4()), + session_id: SipHash.hash!(@hash_key, UUID.uuid4()) } end