Conditionally support switching between v1 and v2 clickhouse schemas (#2780)

* Remove ClickhouseSetup module

This has been an implicit point of contact to many
tests. From now on the goal is for each test to maintain
its own, isolated setup so that no accidental clashes
and implicit assumptions are relied upon.

* Implement v2 schema check

An environment variable V2_MIGRATION_DONE acts like
a feature flag, switching plausible from using old events/sessions
schemas to v2 schemas introduced by NumericIDs migration.

* Run both test suites sequentially

While the code for v1 and v2 schemas must be kept still,
we will from now on run tests against both code paths.
Secondary test run will set V2_MIGRATION_DONE=1 variable,
thus making all `Plausible.v2?()` checks return `true'.

* Remove unused function

This is a remnant from the short period when
we would check for existing events before allowing
creating a new site.

* Update test setups/factories with v2 migration check

* Make GateKeeper return site id along with :allow

* Make Billing module check for v2 schema

* Make ingestion aware of v2 schema

* Disable site transfers for when v2 is live

In a separate changeset we will implement simplified
site transfer for when v2 migration is complete.
The new transfer will only rename the site domain in postgres
and keep track of the original site prior to the transfer
so we keep an ingestion grace period until the customers
redeploy their scripting.

* Make Stats base queries aware of v2 schema switch

* Update breakdown with v2 conditionals

* Update pageview local start with v2 check

* Update current visitoris with v2 check

* Update stats controller with v2 checks

* Update external controller with v2 checks

* Update remaining tests with proper fixtures

* Rewrite redundant assignment

* Remove unused alias

* Mute credo, this is not the right time

* Add test_helper prompt

* Fetch priv dir so it works with a release

* Fetch distinct partitions only

* Don't limit inspect output for partitions

* Ensure SQL is printed to IO

* Remove redundant domain fixture

.github/workflows/elixir.yml

@@ -70,8 +70,12 @@ jobs:
         run: npm run deploy --prefix ./tracker
       - name: Check Credo Warnings
         run: mix credo diff --from-git-merge-base origin/master
-      - name: Run tests
+      - name: Run tests (against v1 schema)
         run: mix test --include slow --max-failures 1 --warnings-as-errors
+      - name: Run tests (against v2 schema)
+        run: mix test --include slow --max-failures 1 --warnings-as-errors
+        env:
+          V2_MIGRATION_DONE: 1
       - name: Check Dialyzer
         run: mix dialyzer
 env:

config/runtime.exs

@@ -99,6 +99,8 @@ ch_db_url =
   |> get_var_from_path_or_env("CLICKHOUSE_MAX_BUFFER_SIZE", "10000")
   |> Integer.parse()
 
+v2_migration_done = get_var_from_path_or_env(config_dir, "V2_MIGRATION_DONE")
+
 ### Mandatory params End
 
 build_metadata_raw = get_var_from_path_or_env(config_dir, "BUILD_METADATA", "{}")
@@ -131,6 +133,8 @@ runtime_metadata = [
 
 config :plausible, :runtime_metadata, runtime_metadata
 
+config :plausible, :v2_migration_done, v2_migration_done
+
 sentry_dsn = get_var_from_path_or_env(config_dir, "SENTRY_DSN")
 honeycomb_api_key = get_var_from_path_or_env(config_dir, "HONEYCOMB_API_KEY")
 honeycomb_dataset = get_var_from_path_or_env(config_dir, "HONEYCOMB_DATASET")

lib/plausible.ex

@@ -6,4 +6,9 @@ defmodule Plausible do
   Contexts are also responsible for managing your data, regardless
   if it comes from the database, an external API or others.
   """
+
+  @spec v2?() :: boolean()
+  def v2?() do
+    Plausible.DataMigration.NumericIDs.ready?()
+  end
 end

lib/plausible/billing/billing.ex

@@ -113,10 +113,28 @@ defmodule Plausible.Billing do
 
   def last_two_billing_months_usage(user, today \\ Timex.today()) do
     {first, second} = last_two_billing_cycles(user, today)
+
+    if Plausible.v2?() do
+      site_ids = Plausible.Sites.owned_site_ids(user)
+
+      usage_for_sites = fn site_ids, date_range ->
+        {pageviews, custom_events} =
+          Plausible.Stats.Clickhouse.usage_breakdown(site_ids, date_range)
+
+        pageviews + custom_events
+      end
+
+      {
+        usage_for_sites.(site_ids, first),
+        usage_for_sites.(site_ids, second)
+      }
+    else
       domains = Plausible.Sites.owned_sites_domains(user)
 
       usage_for_sites = fn domains, date_range ->
-      {pageviews, custom_events} = Plausible.Stats.Clickhouse.usage_breakdown(domains, date_range)
+        {pageviews, custom_events} =
+          Plausible.Stats.Clickhouse.usage_breakdown(domains, date_range)
+
         pageviews + custom_events
       end
 
@@ -125,6 +143,7 @@ defmodule Plausible.Billing do
         usage_for_sites.(domains, second)
       }
     end
+  end
 
   def last_two_billing_cycles(user, today \\ Timex.today()) do
     last_bill_date = user.subscription.last_bill_date
@@ -147,9 +166,14 @@ defmodule Plausible.Billing do
   end
 
   def usage_breakdown(user) do
+    if Plausible.v2?() do
+      site_ids = Plausible.Sites.owned_site_ids(user)
+      Plausible.Stats.Clickhouse.usage_breakdown(site_ids)
+    else
       domains = Plausible.Sites.owned_sites_domains(user)
       Plausible.Stats.Clickhouse.usage_breakdown(domains)
     end
+  end
 
   @doc """
   Returns the number of sites that an account is allowed to have. Accounts for

lib/plausible/data_migration.ex

@@ -13,7 +13,7 @@ defmodule Plausible.DataMigration do
       @repo repo
 
       def run_sql_confirm(name, assigns \\ []) do
-        query = unwrap(name, assigns)
+        query = unwrap_with_io(name, assigns)
 
         confirm("Execute?", fn -> do_run(name, query) end)
       end
@@ -30,7 +30,9 @@ defmodule Plausible.DataMigration do
       end
 
       defp unwrap(name, assigns) do
-        "priv/data_migrations"
+        :plausible
+        |> :code.priv_dir()
+        |> Path.join("data_migrations")
         |> Path.join(@dir)
         |> Path.join("sql")
         |> Path.join(name <> ".sql.eex")

lib/plausible/data_migration/numeric_ids.ex

@@ -7,6 +7,10 @@ defmodule Plausible.DataMigration.NumericIDs do
 
   @table_settings "SETTINGS index_granularity = 8192, storage_policy = 'tiered'"
 
+  def ready?() do
+    Application.get_env(:plausible, :v2_migration_done) || false
+  end
+
   # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity
   def run(opts \\ []) do
     interactive? = Keyword.get(opts, :interactive?, true)
@@ -65,7 +69,7 @@ defmodule Plausible.DataMigration.NumericIDs do
       - table_settings: #{table_settings}
       - db url: #{db_url}
       - cluster?: #{cluster?}
-      - partitions to do: #{inspect(partitions, pretty: true, width: 80)}
+      - partitions to do: #{inspect(partitions, pretty: true, limit: :infinity, width: 80)}
       - start from: #{start_from}
       - stop at: #{stop_at}
     """)

lib/plausible/event/write_buffer.ex

@@ -64,9 +64,14 @@ defmodule Plausible.Event.WriteBuffer do
       events ->
         Logger.info("Flushing #{length(events)} events")
         events = Enum.map(events, &(Map.from_struct(&1) |> Map.delete(:__meta__)))
+
+        if Plausible.v2?() do
+          IngestRepo.insert_all(Plausible.ClickhouseEventV2, events)
+        else
           IngestRepo.insert_all(Plausible.ClickhouseEvent, events)
         end
     end
+  end
 
   defp flush_interval_ms() do
     Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :flush_interval_ms)

lib/plausible/ingestion/event.ex

@@ -7,9 +7,11 @@ defmodule Plausible.Ingestion.Event do
   """
   alias Plausible.Ingestion.Request
   alias Plausible.ClickhouseEvent
+  alias Plausible.ClickhouseEventV2
   alias Plausible.Site.GateKeeper
 
   defstruct domain: nil,
+            site_id: nil,
             clickhouse_event_attrs: %{},
             clickhouse_event: nil,
             dropped?: false,
@@ -26,8 +28,9 @@ defmodule Plausible.Ingestion.Event do
 
   @type t() :: %__MODULE__{
           domain: String.t() | nil,
+          site_id: pos_integer() | nil,
           clickhouse_event_attrs: map(),
-          clickhouse_event: %ClickhouseEvent{} | nil,
+          clickhouse_event: %ClickhouseEvent{} | %ClickhouseEventV2{} | nil,
           dropped?: boolean(),
           drop_reason: drop_reason(),
           request: Request.t(),
@@ -43,10 +46,10 @@ defmodule Plausible.Ingestion.Event do
       else
         Enum.reduce(domains, [], fn domain, acc ->
           case GateKeeper.check(domain) do
-            :allow ->
+            {:allow, site_id} ->
               processed =
                 domain
-                |> new(request)
+                |> new(site_id, request)
                 |> process_unless_dropped(pipeline())
 
               [processed | acc]
@@ -117,6 +120,10 @@ defmodule Plausible.Ingestion.Event do
     struct!(__MODULE__, domain: domain, request: request)
   end
 
+  defp new(domain, site_id, request) do
+    struct!(__MODULE__, domain: domain, site_id: site_id, request: request)
+  end
+
   defp drop(%__MODULE__{} = event, reason, attrs \\ []) do
     fields =
       attrs
@@ -156,6 +163,7 @@ defmodule Plausible.Ingestion.Event do
   defp put_basic_info(%__MODULE__{} = event) do
     update_attrs(event, %{
       domain: event.domain,
+      site_id: event.site_id,
       timestamp: event.request.timestamp,
       name: event.request.event_name,
       hostname: event.request.hostname,
@@ -217,9 +225,15 @@ defmodule Plausible.Ingestion.Event do
 
   defp validate_clickhouse_event(%__MODULE__{} = event) do
     clickhouse_event =
+      if Plausible.v2?() do
+        event
+        |> Map.fetch!(:clickhouse_event_attrs)
+        |> ClickhouseEventV2.new()
+      else
         event
         |> Map.fetch!(:clickhouse_event_attrs)
         |> ClickhouseEvent.new()
+      end
 
     case Ecto.Changeset.apply_action(clickhouse_event, nil) do
       {:ok, valid_clickhouse_event} ->

lib/plausible/session/cache_store.ex

@@ -22,7 +22,14 @@ defmodule Plausible.Session.CacheStore do
   defp find_session(_domain, nil), do: nil
 
   defp find_session(event, user_id) do
-    case Cachex.get(:sessions, {event.domain, user_id}) do
+    from_cache =
+      if Plausible.v2?() do
+        Cachex.get(:sessions, {event.site_id, user_id})
+      else
+        Cachex.get(:sessions, {event.domain, user_id})
+      end
+
+    case from_cache do
       {:ok, nil} ->
         nil
 
@@ -38,7 +45,13 @@ defmodule Plausible.Session.CacheStore do
   end
 
   defp persist_session(session) do
-    key = {session.domain, session.user_id}
+    key =
+      if Plausible.v2?() do
+        {session.site_id, session.user_id}
+      else
+        {session.domain, session.user_id}
+      end
+
     Cachex.put(:sessions, key, session, ttl: :timer.minutes(30))
     session
   end
@@ -68,6 +81,41 @@ defmodule Plausible.Session.CacheStore do
   end
 
   defp new_session_from_event(event) do
+    if Plausible.v2?() do
+      %Plausible.ClickhouseSessionV2{
+        sign: 1,
+        session_id: Plausible.ClickhouseSession.random_uint64(),
+        hostname: event.hostname,
+        site_id: event.site_id,
+        user_id: event.user_id,
+        entry_page: event.pathname,
+        exit_page: event.pathname,
+        is_bounce: true,
+        duration: 0,
+        pageviews: if(event.name == "pageview", do: 1, else: 0),
+        events: 1,
+        referrer: event.referrer,
+        referrer_source: event.referrer_source,
+        utm_medium: event.utm_medium,
+        utm_source: event.utm_source,
+        utm_campaign: event.utm_campaign,
+        utm_content: event.utm_content,
+        utm_term: event.utm_term,
+        country_code: event.country_code,
+        subdivision1_code: event.subdivision1_code,
+        subdivision2_code: event.subdivision2_code,
+        city_geoname_id: event.city_geoname_id,
+        screen_size: event.screen_size,
+        operating_system: event.operating_system,
+        operating_system_version: event.operating_system_version,
+        browser: event.browser,
+        browser_version: event.browser_version,
+        timestamp: event.timestamp,
+        start: event.timestamp,
+        "entry_meta.key": Map.get(event, :"meta.key"),
+        "entry_meta.value": Map.get(event, :"meta.value")
+      }
+    else
       %Plausible.ClickhouseSession{
         sign: 1,
         session_id: Plausible.ClickhouseSession.random_uint64(),
@@ -103,3 +151,4 @@ defmodule Plausible.Session.CacheStore do
       }
     end
   end
+end

lib/plausible/session/write_buffer.ex

@@ -69,9 +69,13 @@ defmodule Plausible.Session.WriteBuffer do
           |> Enum.map(&(Map.from_struct(&1) |> Map.delete(:__meta__)))
           |> Enum.reverse()
 
+        if Plausible.v2?() do
+          IngestRepo.insert_all(Plausible.ClickhouseSessionV2, sessions)
+        else
           IngestRepo.insert_all(Plausible.ClickhouseSession, sessions)
         end
     end
+  end
 
   defp flush_interval_ms() do
     Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :flush_interval_ms)

lib/plausible/site/admin.ex

@@ -84,6 +84,9 @@ defmodule Plausible.SiteAdmin do
   end
 
   def transfer_data([from_site], params) do
+    if Plausible.v2?() do
+      {:error, "Transfers temporarily unspported with v2"}
+    else
       to_site = Repo.get_by(Plausible.Site, domain: params["domain"])
 
       if to_site do
@@ -91,7 +94,9 @@ defmodule Plausible.SiteAdmin do
         {:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, event_q, [], timeout: 30_000)
 
         session_q = session_transfer_query(from_site.domain, to_site.domain)
-      {:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, session_q, [], timeout: 30_000)
+
+        {:ok, _} =
+          Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, session_q, [], timeout: 30_000)
 
         start_date = Plausible.Stats.Clickhouse.pageview_start_date_local(from_site)
 
@@ -106,6 +111,7 @@ defmodule Plausible.SiteAdmin do
         {:error, "Cannot transfer to non-existing domain"}
       end
     end
+  end
 
   def transfer_data(_, _), do: {:error, "Please select exactly one site for this action"}
 

lib/plausible/site/gate_keeper.ex

@@ -1,9 +1,9 @@
 defmodule Plausible.Site.GateKeeper do
   @type policy() :: :allow | :not_found | :block | :throttle
+  @type site_id() :: pos_integer()
   @policy_for_non_existing_sites :not_found
-  @policy_on_rate_limiting_backend_error :allow
 
-  @type t() :: :allow | {:deny, policy()}
+  @type t() :: {:allow, site_id()} | {:deny, policy()}
 
   @moduledoc """
   Thin wrapper around Hammer for gate keeping domain-specific events
@@ -23,7 +23,7 @@ defmodule Plausible.Site.GateKeeper do
   The module defines two policies outside the regular bucket inspection:
     * when the the site is not found in cache: #{@policy_for_non_existing_sites}
     * when the underlying rate limiting mechanism returns
-      an internal error: #{@policy_on_rate_limiting_backend_error}
+      an internal error: :allow
   """
   alias Plausible.Site
   alias Plausible.Site.Cache
@@ -33,7 +33,7 @@ defmodule Plausible.Site.GateKeeper do
   @spec check(String.t(), Keyword.t()) :: t()
   def check(domain, opts \\ []) when is_binary(domain) do
     case policy(domain, opts) do
-      :allow -> :allow
+      {:allow, site_id} -> {:allow, site_id}
       other -> {:deny, other}
     end
   end
@@ -56,15 +56,15 @@ defmodule Plausible.Site.GateKeeper do
     result
   end
 
-  defp check_rate_limit(%Site{ingest_rate_limit_threshold: nil}, _opts) do
+  defp check_rate_limit(%Site{id: site_id, ingest_rate_limit_threshold: nil}, _opts) do
-    :allow
+    {:allow, site_id}
   end
 
   defp check_rate_limit(%Site{ingest_rate_limit_threshold: 0}, _opts) do
     :block
   end
 
-  defp check_rate_limit(%Site{ingest_rate_limit_threshold: threshold} = site, opts)
+  defp check_rate_limit(%Site{id: site_id, ingest_rate_limit_threshold: threshold} = site, opts)
        when is_integer(threshold) do
     key = Keyword.get(opts, :key, key(site.domain))
     scale_ms = site.ingest_rate_limit_scale_seconds * 1_000
@@ -74,14 +74,14 @@ defmodule Plausible.Site.GateKeeper do
         :throttle
 
       {:allow, _} ->
-        :allow
+        {:allow, site_id}
 
       {:error, reason} ->
         Logger.error(
-          "Error checking rate limit for '#{key}': #{inspect(reason)}. Falling back to: #{@policy_on_rate_limiting_backend_error}"
+          "Error checking rate limit for '#{key}': #{inspect(reason)}. Falling back to: :allow"
         )
 
-        @policy_on_rate_limiting_backend_error
+        {:allow, site_id}
     end
   end
 end

lib/plausible/sites.ex

@@ -1,6 +1,5 @@
 defmodule Plausible.Sites do
   use Plausible.Repo
-  alias Plausible.ClickhouseRepo
   alias Plausible.Site
   alias Plausible.Site.SharedLink
   import Ecto.Query
@@ -77,11 +76,6 @@ defmodule Plausible.Sites do
     !!stats_start_date(site)
   end
 
-  def has_events?(domain) do
-    q = from e in "events", where: e.domain == ^domain, select: true, limit: 1
-    ClickhouseRepo.one(q) == true
-  end
-
   def create_shared_link(site, name, password \\ nil) do
     changes =
       SharedLink.changeset(
@@ -158,6 +152,13 @@ defmodule Plausible.Sites do
     |> Repo.all()
   end
 
+  def owned_site_ids(user) do
+    user
+    |> owned_sites_query()
+    |> select([site], site.id)
+    |> Repo.all()
+  end
+
   defp owned_sites_query(user) do
     from(s in Site,
       join: sm in Site.Membership,

lib/plausible/stats/base.ex

@@ -35,11 +35,19 @@ defmodule Plausible.Stats.Base do
     {first_datetime, last_datetime} = utc_boundaries(query, site)
 
     q =
+      if Plausible.v2?() do
+        from(
+          e in "events_v2",
+          where: e.site_id == ^site.id,
+          where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
+        )
+      else
         from(
           e in "events",
           where: e.domain == ^site.domain,
           where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
         )
+      end
       |> add_sample_hint(query)
 
     q =
@@ -190,11 +198,19 @@ defmodule Plausible.Stats.Base do
     {first_datetime, last_datetime} = utc_boundaries(query, site)
 
     sessions_q =
+      if Plausible.v2?() do
+        from(
+          s in "sessions_v2",
+          where: s.site_id == ^site.id,
+          where: s.start >= ^first_datetime and s.start < ^last_datetime
+        )
+      else
         from(
           s in "sessions",
           where: s.domain == ^site.domain,
           where: s.start >= ^first_datetime and s.start < ^last_datetime
         )
+      end
       |> add_sample_hint(query)
       |> filter_by_entry_props(query)
 

lib/plausible/stats/breakdown.ex

@@ -312,7 +312,7 @@ defmodule Plausible.Stats.Breakdown do
   end
 
   defp do_group_by(
-         %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events", _}}} = q,
+         %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,
          "event:props:" <> prop
        ) do
     q =
@@ -336,7 +336,7 @@ defmodule Plausible.Stats.Breakdown do
   end
 
   defp do_group_by(
-         %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events", _}}} = q,
+         %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,
          "event:name"
        ) do
     from(
@@ -348,7 +348,7 @@ defmodule Plausible.Stats.Breakdown do
   end
 
   defp do_group_by(
-         %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events", _}}} = q,
+         %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,
          "event:page"
        ) do
     from(
@@ -360,7 +360,7 @@ defmodule Plausible.Stats.Breakdown do
   end
 
   defp do_group_by(
-         %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events", _}}} = q,
+         %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,
          "event:page_match"
        ) do
     case Map.get(q, :__private_match_sources__) do

lib/plausible/stats/clickhouse.ex

@@ -8,12 +8,23 @@ defmodule Plausible.Stats.Clickhouse do
   @spec pageview_start_date_local(Plausible.Site.t()) :: Date.t() | nil
   def pageview_start_date_local(site) do
     datetime =
+      if Plausible.v2?() do
         ClickhouseRepo.one(
-        from e in "events",
+          from(e in "events_v2",
+            select: fragment("min(?)", e.timestamp),
+            where: e.site_id == ^site.id,
+            where: e.timestamp >= ^site.native_stats_start_at
+          )
+        )
+      else
+        ClickhouseRepo.one(
+          from(e in "events",
             select: fragment("min(?)", e.timestamp),
             where: e.domain == ^site.domain,
             where: e.timestamp >= ^site.native_stats_start_at
           )
+        )
+      end
 
     case datetime do
       # no stats for this domain yet
@@ -29,28 +40,29 @@ defmodule Plausible.Stats.Clickhouse do
 
   def imported_pageview_count(site) do
     Plausible.ClickhouseRepo.one(
-      from i in "imported_visitors",
+      from(i in "imported_visitors",
         where: i.site_id == ^site.id,
         select: sum(i.pageviews)
       )
+    )
   end
 
-  def usage_breakdown(domains) do
+  def usage_breakdown(domains_or_site_ids) do
     range =
       Date.range(
         Timex.shift(Timex.today(), days: -30),
         Timex.today()
       )
 
-    usage_breakdown(domains, range)
+    usage_breakdown(domains_or_site_ids, range)
   end
 
-  def usage_breakdown(domains, date_range) do
+  def usage_breakdown([d | _] = domains, date_range) when is_binary(d) do
     Enum.chunk_every(domains, 300)
     |> Enum.reduce({0, 0}, fn domains, {pageviews_total, custom_events_total} ->
       {chunk_pageviews, chunk_custom_events} =
         ClickhouseRepo.one(
-          from e in "events",
+          from(e in "events",
             where: e.domain in ^domains,
             where: fragment("toDate(?)", e.timestamp) >= ^date_range.first,
             where: fragment("toDate(?)", e.timestamp) <= ^date_range.last,
@@ -59,11 +71,34 @@ defmodule Plausible.Stats.Clickhouse do
               fragment("countIf(? != 'pageview')", e.name)
             }
           )
+        )
 
       {pageviews_total + chunk_pageviews, custom_events_total + chunk_custom_events}
     end)
   end
 
+  def usage_breakdown([sid | _] = site_ids, date_range) when is_integer(sid) do
+    Enum.chunk_every(site_ids, 300)
+    |> Enum.reduce({0, 0}, fn site_ids, {pageviews_total, custom_events_total} ->
+      {chunk_pageviews, chunk_custom_events} =
+        ClickhouseRepo.one(
+          from(e in "events_v2",
+            where: e.site_id in ^site_ids,
+            where: fragment("toDate(?)", e.timestamp) >= ^date_range.first,
+            where: fragment("toDate(?)", e.timestamp) <= ^date_range.last,
+            select: {
+              fragment("countIf(? = 'pageview')", e.name),
+              fragment("countIf(? != 'pageview')", e.name)
+            }
+          )
+        )
+
+      {pageviews_total + chunk_pageviews, custom_events_total + chunk_custom_events}
+    end)
+  end
+
+  def usage_breakdown([], _date_range), do: {0, 0}
+
   def top_sources(site, query, limit, page, show_noref \\ false, include_details) do
     offset = (page - 1) * limit
 
@@ -151,12 +186,24 @@ defmodule Plausible.Stats.Clickhouse do
 
   def current_visitors(site, query) do
     Plausible.ClickhouseRepo.one(
-      from e in base_query(site, query),
+      from(e in base_query(site, query),
         select: uniq(e.user_id)
       )
+    )
   end
 
   def has_pageviews?(site) do
+    if Plausible.v2?() do
+      ClickhouseRepo.exists?(
+        from(e in "events_v2",
+          where:
+            e.site_id == ^site.id and
+              e.name == "pageview" and
+              e.timestamp >=
+                ^site.native_stats_start_at
+        )
+      )
+    else
       ClickhouseRepo.exists?(
         from(e in "events",
           where:
@@ -167,31 +214,59 @@ defmodule Plausible.Stats.Clickhouse do
         )
       )
     end
+  end
 
   def last_24h_visitors([]), do: %{}
 
   def last_24h_visitors(sites) do
+    if Plausible.v2?() do
+      site_id_to_domain_mapping = for site <- sites, do: {site.id, site.domain}, into: %{}
+
+      ClickhouseRepo.all(
+        from(e in "events_v2",
+          group_by: e.site_id,
+          where: e.site_id in ^Map.keys(site_id_to_domain_mapping),
+          where: e.timestamp > fragment("now() - INTERVAL 24 HOUR"),
+          select: {e.site_id, fragment("uniq(user_id)")}
+        )
+      )
+      |> Enum.map(fn {site_id, user_id} ->
+        {site_id_to_domain_mapping[site_id], user_id}
+      end)
+      |> Enum.into(%{})
+    else
       domains = Enum.map(sites, & &1.domain)
 
       ClickhouseRepo.all(
-      from e in "events",
+        from(e in "events",
           group_by: e.domain,
           where: e.domain in ^domains,
           where: e.timestamp > fragment("now() - INTERVAL 24 HOUR"),
           select: {e.domain, fragment("uniq(user_id)")}
         )
+      )
       |> Enum.into(%{})
     end
+  end
 
+  # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity
   defp base_session_query(site, query) do
     {first_datetime, last_datetime} = utc_boundaries(query, site)
 
     q =
+      if Plausible.v2?() do
+        from(s in "sessions_v2",
+          hints: ["SAMPLE 10000000"],
+          where: s.site_id == ^site.id,
+          where: s.timestamp >= ^first_datetime and s.start < ^last_datetime
+        )
+      else
         from(s in "sessions",
           hints: ["SAMPLE 10000000"],
           where: s.domain == ^site.domain,
           where: s.timestamp >= ^first_datetime and s.start < ^last_datetime
         )
+      end
 
     q =
       if query.filters["source"] do
@@ -302,15 +377,24 @@ defmodule Plausible.Stats.Clickhouse do
     end
   end
 
+  # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity
   defp base_query_bare(site, query) do
     {first_datetime, last_datetime} = utc_boundaries(query, site)
 
     q =
+      if Plausible.v2?() do
+        from(e in "events_v2",
+          hints: ["SAMPLE 10000000"],
+          where: e.site_id == ^site.id,
+          where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
+        )
+      else
         from(e in "events",
           hints: ["SAMPLE 10000000"],
           where: e.domain == ^site.domain,
           where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
         )
+      end
 
     q =
       if query.filters["screen"] do

lib/plausible/stats/current_visitors.ex

@@ -8,6 +8,14 @@ defmodule Plausible.Stats.CurrentVisitors do
       |> Timex.shift(minutes: -5)
       |> NaiveDateTime.truncate(:second)
 
+    if Plausible.v2?() do
+      ClickhouseRepo.one(
+        from e in "events_v2",
+          where: e.site_id == ^site.id,
+          where: e.timestamp >= ^first_datetime,
+          select: uniq(e.user_id)
+      )
+    else
       ClickhouseRepo.one(
         from e in "events",
           where: e.domain == ^site.domain,
@@ -16,3 +24,4 @@ defmodule Plausible.Stats.CurrentVisitors do
       )
     end
   end
+end

lib/plausible_web/controllers/api/stats_controller.ex

@@ -1161,9 +1161,14 @@ defmodule PlausibleWeb.Api.StatsController do
     site = conn.assigns[:site]
 
     with :ok <- validate_params(params) do
-      query = Query.from(site, params) |> Filters.add_prefix()
+      query =
+        Query.from(site, params)
+        |> Filters.add_prefix()
 
-      json(conn, Stats.filter_suggestions(site, query, params["filter_name"], params["q"]))
+      json(
+        conn,
+        Stats.filter_suggestions(site, query, params["filter_name"], params["q"])
+      )
     else
       {:error, message} when is_binary(message) -> bad_request(conn, message)
     end

mix.lock

@@ -10,7 +10,7 @@
   "cachex": {:hex, :cachex, "3.4.0", "868b2959ea4aeb328c6b60ff66c8d5123c083466ad3c33d3d8b5f142e13101fb", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "370123b1ab4fba4d2965fb18f87fd758325709787c8c5fce35b3fe80645ccbe5"},
   "castore": {:hex, :castore, "0.1.22", "4127549e411bedd012ca3a308dede574f43819fe9394254ca55ab4895abfa1a2", [:mix], [], "hexpm", "c17576df47eb5aa1ee40cc4134316a99f5cad3e215d5c77b8dd3cfef12a22cac"},
   "certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"},
-  "ch": {:git, "https://github.com/ruslandoga/ch.git", "5894f81e246c049b9db5186497189cce226947aa", []},
+  "ch": {:git, "https://github.com/plausible/ch.git", "f411daa07c6310d3308f81397905df65330aeb64", []},
   "chatterbox": {:hex, :ts_chatterbox, "0.13.0", "6f059d97bcaa758b8ea6fffe2b3b81362bd06b639d3ea2bb088335511d691ebf", [:rebar3], [{:hpack, "~> 0.2.3", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "b93d19104d86af0b3f2566c4cba2a57d2e06d103728246ba1ac6c3c0ff010aa7"},
   "chto": {:git, "https://github.com/plausible/chto.git", "316421ede947541bac32a853d70b0ca8ba003220", []},
   "combination": {:hex, :combination, "0.0.3", "746aedca63d833293ec6e835aa1f34974868829b1486b1e1cb0685f0b2ae1f41", [:mix], [], "hexpm", "72b099f463df42ef7dc6371d250c7070b57b6c5902853f69deb894f79eda18ca"},

priv/data_migrations/NumericIDs/sql/list-partitions.sql.eex

@@ -1,4 +1,4 @@
-SELECT partition 
+SELECT distinct(partition)
 FROM system.parts 
 WHERE table = 'events' 
 <%= if @start_from do %>AND partition >= '<%= @start_from %>'<% end %>

test/plausible/auth/auth_test.exs

@@ -18,7 +18,11 @@ defmodule Plausible.AuthTest do
 
     test "is true if user does have events" do
       user = insert(:user)
-      insert(:site, members: [user], domain: "test-site.com")
+      site = insert(:site, members: [user])
+
+      populate_stats(site, [
+        build(:pageview)
+      ])
 
       assert Auth.has_active_sites?(user)
     end

test/plausible/billing/billing_test.exs

@@ -133,10 +133,10 @@ defmodule Plausible.BillingTest do
       site = insert(:site, members: [user])
 
       create_pageviews([
-        %{domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]},
+        %{site: site, timestamp: ~N[2021-01-01 00:00:00]},
-        %{domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]},
+        %{site: site, timestamp: ~N[2020-12-31 00:00:00]},
-        %{domain: site.domain, timestamp: ~N[2020-11-01 00:00:00]},
+        %{site: site, timestamp: ~N[2020-11-01 00:00:00]},
-        %{domain: site.domain, timestamp: ~N[2020-10-31 00:00:00]}
+        %{site: site, timestamp: ~N[2020-10-31 00:00:00]}
       ])
 
       assert Billing.last_two_billing_months_usage(user, today) == {1, 1}
@@ -163,10 +163,10 @@ defmodule Plausible.BillingTest do
         )
 
       create_pageviews([
-        %{domain: owner_site.domain, timestamp: ~N[2020-12-31 00:00:00]},
+        %{site: owner_site, timestamp: ~N[2020-12-31 00:00:00]},
-        %{domain: admin_site.domain, timestamp: ~N[2020-12-31 00:00:00]},
+        %{site: admin_site, timestamp: ~N[2020-12-31 00:00:00]},
-        %{domain: owner_site.domain, timestamp: ~N[2020-11-01 00:00:00]},
+        %{site: owner_site, timestamp: ~N[2020-11-01 00:00:00]},
-        %{domain: admin_site.domain, timestamp: ~N[2020-11-01 00:00:00]}
+        %{site: admin_site, timestamp: ~N[2020-11-01 00:00:00]}
       ])
 
       assert Billing.last_two_billing_months_usage(user, today) == {1, 1}

test/plausible/session/cache_store_test.exs

@@ -37,7 +37,13 @@ defmodule Plausible.Session.CacheStoreTest do
     assert_receive({WriteBuffer, :insert, [sessions]})
     assert [session] = sessions
     assert session.hostname == event.hostname
+
+    if Plausible.v2?() do
+      assert session.site_id == event.site_id
+    else
       assert session.domain == event.domain
+    end
+
     assert session.user_id == event.user_id
     assert session.entry_page == event.pathname
     assert session.exit_page == event.pathname
@@ -68,12 +74,9 @@ defmodule Plausible.Session.CacheStoreTest do
     timestamp = Timex.now()
     event1 = build(:event, name: "pageview", timestamp: timestamp |> Timex.shift(seconds: -10))
 
-    event2 =
+    event2 = %{
-      build(:event,
+      event1
-        domain: event1.domain,
+      | timestamp: timestamp,
-        user_id: event1.user_id,
-        name: "pageview",
-        timestamp: timestamp,
         country_code: "US",
         subdivision1_code: "SUB1",
         subdivision2_code: "SUB2",
@@ -83,7 +86,7 @@ defmodule Plausible.Session.CacheStoreTest do
         operating_system_version: "11",
         browser: "Firefox",
         browser_version: "10"
-      )
+    }
 
     CacheStore.on_event(event1, nil, buffer)
     CacheStore.on_event(event2, nil, buffer)
@@ -107,13 +110,7 @@ defmodule Plausible.Session.CacheStoreTest do
     timestamp = Timex.now()
     event1 = build(:event, name: "pageview", timestamp: timestamp |> Timex.shift(seconds: 10))
 
-    event2 =
+    event2 = %{event1 | timestamp: timestamp}
-      build(:event,
-        domain: event1.domain,
-        user_id: event1.user_id,
-        name: "pageview",
-        timestamp: timestamp
-      )
 
     CacheStore.on_event(event1, nil, buffer)
     CacheStore.on_event(event2, nil, buffer)

test/plausible/site/admin_test.exs

@@ -1,10 +1,18 @@
 defmodule Plausible.SiteAdminTest do
   use Plausible.DataCase, async: true
 
-  alias Plausible.{SiteAdmin, ClickhouseRepo, ClickhouseEvent, ClickhouseSession}
+  alias Plausible.{
+    SiteAdmin,
+    ClickhouseRepo,
+    ClickhouseEvent,
+    ClickhouseSession
+  }
 
   describe "transfer_data" do
     test "event and session structs remain the same after transfer" do
+      if Plausible.v2?() do
+        :ok
+      else
         from_site = insert(:site)
         to_site = insert(:site)
 
@@ -18,13 +26,18 @@ defmodule Plausible.SiteAdminTest do
         event_after = get_event_by_domain(to_site.domain)
         session_after = get_session_by_domain(to_site.domain)
 
+        assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}
         assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}
         assert session_before == %ClickhouseSession{session_after | transferred_from: ""}
         assert event_after.transferred_from == from_site.domain
         assert session_after.transferred_from == from_site.domain
       end
+    end
 
     test "transfers all events and sessions" do
+      if Plausible.v2?() do
+        :ok
+      else
         from_site = insert(:site)
         to_site = insert(:site)
 
@@ -50,8 +63,12 @@ defmodule Plausible.SiteAdminTest do
         assert length(transferred_events) == 4
         assert length(transferred_sessions) == 3
       end
+    end
 
     test "updates stats_start_date on site record" do
+      if Plausible.v2?() do
+        :ok
+      else
         from_site = insert(:site)
         to_site = insert(:site)
 
@@ -61,8 +78,12 @@ defmodule Plausible.SiteAdminTest do
 
         assert Repo.reload(to_site).stats_start_date == ~D[2022-01-01]
       end
+    end
 
     test "updates native_stats_start_date on based on the from_site record" do
+      if Plausible.v2?() do
+        :ok
+      else
         from_site = insert(:site, native_stats_start_at: ~N[2022-01-01 01:00:00])
         to_site = insert(:site)
 
@@ -72,6 +93,7 @@ defmodule Plausible.SiteAdminTest do
 
         assert Repo.reload(to_site).native_stats_start_at == ~N[2022-01-01 01:00:00]
       end
+    end
 
     test "session_transfer_query" do
       actual = SiteAdmin.session_transfer_query("from.com", "to.com")

test/plausible/site/gate_keeper_test.exs

@@ -19,20 +19,21 @@ defmodule Plausible.Site.GateKeeperTest do
   test "site from cache with no ingest_rate_limit_threshold is allowed", %{test: test, opts: opts} do
     domain = "site1.example.com"
 
-    add_site_and_refresh_cache(test, domain: domain)
+    %{id: site_id} = add_site_and_refresh_cache(test, domain: domain)
-    assert :allow = GateKeeper.check(domain, opts)
+    assert {:allow, ^site_id} = GateKeeper.check(domain, opts)
   end
 
   test "rate limiting works with threshold", %{test: test, opts: opts} do
     domain = "site1.example.com"
 
+    %{id: site_id} =
       add_site_and_refresh_cache(test,
         domain: domain,
         ingest_rate_limit_threshold: 1,
         ingest_rate_limit_scale_seconds: 60
       )
 
-    assert :allow = GateKeeper.check(domain, opts)
+    assert {:allow, ^site_id} = GateKeeper.check(domain, opts)
     assert {:deny, :throttle} = GateKeeper.check(domain, opts)
     assert {:deny, :throttle} = GateKeeper.check(domain, opts)
   end
@@ -40,17 +41,18 @@ defmodule Plausible.Site.GateKeeperTest do
   test "rate limiting works with scale window", %{test: test, opts: opts} do
     domain = "site1.example.com"
 
+    %{id: site_id} =
       add_site_and_refresh_cache(test,
         domain: domain,
         ingest_rate_limit_threshold: 1,
         ingest_rate_limit_scale_seconds: 1
       )
 
-    assert :allow = GateKeeper.check(domain, opts)
+    assert {:allow, ^site_id} = GateKeeper.check(domain, opts)
     Process.sleep(1)
     assert {:deny, :throttle} = GateKeeper.check(domain, opts)
     Process.sleep(1_000)
-    assert :allow = GateKeeper.check(domain, opts)
+    assert {:allow, ^site_id} = GateKeeper.check(domain, opts)
   end
 
   test "rate limiting prioritises cache lookups", %{test: test, opts: opts} do
@@ -67,8 +69,9 @@ defmodule Plausible.Site.GateKeeperTest do
     # We need some dummy site, otherwise the cache won't refresh in case the DB
     # is completely empty
     insert(:site)
+    deleted_site_id = site.id
 
-    assert :allow = GateKeeper.check(domain, opts)
+    assert {:allow, ^deleted_site_id} = GateKeeper.check(domain, opts)
     :ok = Cache.refresh_all(opts[:cache_opts])
     assert {:deny, :not_found} = GateKeeper.check(domain, opts)
   end
@@ -86,18 +89,19 @@ defmodule Plausible.Site.GateKeeperTest do
         ingest_rate_limit_scale_seconds: 600
       )
 
-    assert :allow = GateKeeper.check(domain, opts)
+    site_id = site.id
+    assert {:allow, ^site_id} = GateKeeper.check(domain, opts)
     assert {:deny, :throttle} = GateKeeper.check(domain, opts)
 
     {:ok, :broken} = break_hammer(site)
 
     log =
       capture_log(fn ->
-        assert :allow = GateKeeper.check(domain, opts)
+        assert {:allow, ^site_id} = GateKeeper.check(domain, opts)
       end)
 
     assert log =~ "Error checking rate limit for 'ingest:site:causingerrors.example.com'"
-    assert log =~ "Falling back to: allow"
+    assert log =~ "Falling back to: :allow"
   end
 
   # We need a way to force Hammer to error-out on Hammer.check_rate/3.

test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs

@@ -97,10 +97,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
   end
 
   test "aggregates a single metric", %{conn: conn, site: site} do
-    populate_stats([
+    populate_stats(site, [
-      build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
+      build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),
-      build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
+      build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]),
-      build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+      build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
     ])
 
     conn =
@@ -120,12 +120,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
     conn: conn,
     site: site
   } do
-    populate_stats([
+    populate_stats(site, [
-      build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
+      build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),
-      build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
+      build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]),
-      build(:pageview, user_id: 456, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
+      build(:pageview, user_id: 456, timestamp: ~N[2021-01-01 00:00:00]),
-      build(:pageview, user_id: 456, domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
+      build(:pageview, user_id: 456, timestamp: ~N[2021-01-01 00:25:00]),
-      build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+      build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
     ])
 
     conn =
@@ -145,10 +145,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
     conn: conn,
     site: site
   } do
-    populate_stats([
+    populate_stats(site, [
-      build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
+      build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),
-      build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
+      build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]),
-      build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+      build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
     ])
 
     conn =
@@ -171,19 +171,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
 
   describe "comparisons" do
     test "compare period=day with previous period", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
-        build(:pageview, domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]),
+        build(:pageview, timestamp: ~N[2020-12-31 00:00:00]),
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
       ])
 
       conn =
@@ -204,19 +202,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
     end
 
     test "compare period=6mo with previous period", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
-        build(:pageview, domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]),
+        build(:pageview, timestamp: ~N[2020-12-31 00:00:00]),
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-02-01 00:25:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-03-01 00:00:00])
+        build(:pageview, timestamp: ~N[2021-03-01 00:00:00])
       ])
 
       conn =
@@ -239,19 +235,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
 
   describe "filters" do
     test "can filter by source", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           referrer_source: "Google",
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
       ])
 
       conn =
@@ -272,20 +266,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
     end
 
     test "can filter by no source/referrer", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
         build(:pageview,
           referrer_source: "Google",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         )
       ])
@@ -308,19 +299,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
     end
 
     test "can filter by referrer", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           referrer: "https://facebook.com",
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
       ])
 
       conn =
@@ -341,19 +330,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
     end
 
     test "can filter by utm_medium", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           utm_medium: "social",
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
       ])
 
       conn =
@@ -374,19 +361,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
     end
 
     test "can filter by utm_source", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           utm_source: "Twitter",
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
       ])
 
       conn =
@@ -407,19 +392,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
     end
 
     test "can filter by utm_campaign", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           utm_campaign: "profile",
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
       ])
 
       conn =
@@ -440,19 +423,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
     end
 
     test "can filter by device type", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           screen_size: "Desktop",
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
       ])
 
       conn =
@@ -473,19 +454,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
     end
 
     test "can filter by browser", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           browser: "Chrome",
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
       ])
 
       conn =
@@ -506,20 +485,18 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
     end
 
     test "can filter by browser version", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           browser: "Chrome",
           browser_version: "56",
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
       ])
 
       conn =
@@ -540,19 +517,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
     end
 
     test "can filter by operating system", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           operating_system: "Mac",
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
       ])
 
       conn =
@@ -573,19 +548,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
     end
 
     test "can filter by operating system version", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           operating_system_version: "10.5",
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
       ])
 
       conn =
@@ -606,19 +579,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
     end
 
     test "can filter by country", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           country_code: "EE",
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
       ])
 
       conn =
@@ -642,22 +613,19 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
       conn: conn,
       site: site
     } do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           user_id: @user_id,
           pathname: "/blogpost",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
         build(:pageview,
           pathname: "/blogpost",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         )
       ])
@@ -680,25 +648,22 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
     end
 
     test "filtering by event:name", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:event,
           name: "Signup",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:event,
           name: "Signup",
-          domain: site.domain,
           user_id: @user_id,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
         build(:event,
           name: "Signup",
-          domain: site.domain,
           user_id: @user_id,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
       ])
 
       conn =
@@ -717,23 +682,20 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
     end
 
     test "combining filters", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           pathname: "/blogpost",
           country_code: "EE",
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
         build(:pageview,
           pathname: "/blogpost",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         )
       ])
@@ -830,15 +792,54 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
       site: site
     } do
       create_sessions([
-        %{domain: site.domain, session_id: 1000, country_code: "EE", sign: 1, events: 1},
+        %{
-        %{domain: site.domain, session_id: 1000, country_code: "EE", sign: -1, events: 1},
+          domain: site.domain,
-        %{domain: site.domain, session_id: 1000, country_code: "EE", sign: 1, events: 2}
+          site_id: site.id,
+          session_id: 1000,
+          country_code: "EE",
+          sign: 1,
+          events: 1
+        },
+        %{
+          domain: site.domain,
+          site_id: site.id,
+          session_id: 1000,
+          country_code: "EE",
+          sign: -1,
+          events: 1
+        },
+        %{
+          domain: site.domain,
+          site_id: site.id,
+          session_id: 1000,
+          country_code: "EE",
+          sign: 1,
+          events: 2
+        }
       ])
 
       create_events([
-        %{domain: site.domain, session_id: 1000, country_code: "EE", name: "pageview"},
+        %{
-        %{domain: site.domain, session_id: 1000, country_code: "EE", name: "pageview"},
+          domain: site.domain,
-        %{domain: site.domain, session_id: 1000, country_code: "EE", name: "pageview"}
+          site_id: site.id,
+          session_id: 1000,
+          country_code: "EE",
+          name: "pageview"
+        },
+        %{
+          domain: site.domain,
+          site_id: site.id,
+          session_id: 1000,
+          country_code: "EE",
+          name: "pageview"
+        },
+        %{
+          domain: site.domain,
+          site_id: site.id,
+          session_id: 1000,
+          country_code: "EE",
+          name: "pageview"
+        }
       ])
 
       conn =

test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs

@@ -131,20 +131,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
   end
 
   test "breakdown by visit:source", %{conn: conn, site: site} do
-    populate_stats([
+    populate_stats(site, [
       build(:pageview,
         referrer_source: "Google",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:00:00]
       ),
       build(:pageview,
         referrer_source: "Google",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:25:00]
       ),
       build(:pageview,
         referrer_source: "",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:00:00]
       )
     ])
@@ -166,10 +163,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
   end
 
   test "breakdown by visit:country", %{conn: conn, site: site} do
-    populate_stats([
+    populate_stats(site, [
-      build(:pageview, country_code: "EE", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
+      build(:pageview, country_code: "EE", timestamp: ~N[2021-01-01 00:00:00]),
-      build(:pageview, country_code: "EE", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
+      build(:pageview, country_code: "EE", timestamp: ~N[2021-01-01 00:25:00]),
-      build(:pageview, country_code: "US", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+      build(:pageview, country_code: "US", timestamp: ~N[2021-01-01 00:00:00])
     ])
 
     conn =
@@ -189,20 +186,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
   end
 
   test "breakdown by visit:referrer", %{conn: conn, site: site} do
-    populate_stats([
+    populate_stats(site, [
       build(:pageview,
         referrer: "https://ref.com",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:00:00]
       ),
       build(:pageview,
         referrer: "https://ref.com",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:25:00]
       ),
       build(:pageview,
         referrer: "",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:00:00]
       )
     ])
@@ -224,20 +218,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
   end
 
   test "breakdown by visit:utm_medium", %{conn: conn, site: site} do
-    populate_stats([
+    populate_stats(site, [
       build(:pageview,
         utm_medium: "Search",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:00:00]
       ),
       build(:pageview,
         utm_medium: "Search",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:25:00]
       ),
       build(:pageview,
         utm_medium: "",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:00:00]
       )
     ])
@@ -259,20 +250,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
   end
 
   test "breakdown by visit:utm_source", %{conn: conn, site: site} do
-    populate_stats([
+    populate_stats(site, [
       build(:pageview,
         utm_source: "Google",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:00:00]
       ),
       build(:pageview,
         utm_source: "Google",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:25:00]
       ),
       build(:pageview,
         utm_source: "",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:00:00]
       )
     ])
@@ -390,20 +378,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
   end
 
   test "breakdown by visit:device", %{conn: conn, site: site} do
-    populate_stats([
+    populate_stats(site, [
       build(:pageview,
         screen_size: "Desktop",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:00:00]
       ),
       build(:pageview,
         screen_size: "Desktop",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:25:00]
       ),
       build(:pageview,
         screen_size: "Mobile",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:00:00]
       )
     ])
@@ -425,20 +410,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
   end
 
   test "breakdown by visit:os", %{conn: conn, site: site} do
-    populate_stats([
+    populate_stats(site, [
       build(:pageview,
         operating_system: "Mac",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:00:00]
       ),
       build(:pageview,
         operating_system: "Mac",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:25:00]
       ),
       build(:pageview,
         operating_system: "Windows",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:00:00]
       )
     ])
@@ -460,20 +442,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
   end
 
   test "breakdown by visit:os_version", %{conn: conn, site: site} do
-    populate_stats([
+    populate_stats(site, [
       build(:pageview,
         operating_system_version: "10.5",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:00:00]
       ),
       build(:pageview,
         operating_system_version: "10.5",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:25:00]
       ),
       build(:pageview,
         operating_system_version: "10.6",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:00:00]
       )
     ])
@@ -495,10 +474,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
   end
 
   test "breakdown by visit:browser", %{conn: conn, site: site} do
-    populate_stats([
+    populate_stats(site, [
-      build(:pageview, browser: "Firefox", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
+      build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]),
-      build(:pageview, browser: "Firefox", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
+      build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:25:00]),
-      build(:pageview, browser: "Safari", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+      build(:pageview, browser: "Safari", timestamp: ~N[2021-01-01 00:00:00])
     ])
 
     conn =
@@ -518,20 +497,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
   end
 
   test "breakdown by visit:browser_version", %{conn: conn, site: site} do
-    populate_stats([
+    populate_stats(site, [
       build(:pageview,
         browser_version: "56",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:00:00]
       ),
       build(:pageview,
         browser_version: "56",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:25:00]
       ),
       build(:pageview,
         browser_version: "57",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:00:00]
       )
     ])
@@ -553,12 +529,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
   end
 
   test "breakdown by event:page", %{conn: conn, site: site} do
-    populate_stats([
+    populate_stats(site, [
-      build(:pageview, pathname: "/", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
+      build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),
-      build(:pageview, pathname: "/", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
+      build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),
       build(:pageview,
         pathname: "/plausible.io",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:00:00]
       )
     ])
@@ -583,12 +558,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
     conn: conn,
     site: site
   } do
-    populate_stats([
+    populate_stats(site, [
-      build(:pageview, pathname: "/", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
+      build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),
-      build(:pageview, pathname: "/", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
+      build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),
       build(:pageview,
         pathname: "/plausible.io",
-        domain: site.domain,
         timestamp: ~N[2021-01-01 00:00:00]
       )
     ])
@@ -609,19 +583,16 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
 
   describe "custom events" do
     test "can breakdown by event:name", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:event,
           name: "Signup",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:event,
           name: "Signup",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         )
       ])
@@ -643,40 +614,34 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
     end
 
     test "can breakdown by event:name with visitors and events metrics", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
-          domain: site.domain,
           pathname: "/non-existing",
           user_id: @user_id,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:event,
           name: "404",
-          domain: site.domain,
           pathname: "/non-existing",
           user_id: @user_id,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
-          domain: site.domain,
           pathname: "/non-existing",
           timestamp: ~N[2021-01-01 00:00:01]
         ),
         build(:pageview,
-          domain: site.domain,
           pathname: "/non-existing",
           user_id: @user_id,
           timestamp: ~N[2021-01-01 00:00:02]
         ),
         build(:event,
           name: "404",
-          domain: site.domain,
           pathname: "/non-existing",
           user_id: @user_id,
           timestamp: ~N[2021-01-01 00:00:02]
         ),
         build(:pageview,
-          domain: site.domain,
           pathname: "/non-existing",
           user_id: @user_id,
           timestamp: ~N[2021-01-01 00:00:03]
@@ -701,39 +666,34 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
     end
 
     test "can breakdown by event:name while filtering for something", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:event,
           name: "Signup",
           pathname: "/pageA",
           browser: "Chrome",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:event,
           name: "Signup",
           pathname: "/pageA",
           browser: "Chrome",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:event,
           name: "Signup",
           pathname: "/pageA",
           browser: "Safari",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:event,
           name: "Signup",
           pathname: "/pageB",
           browser: "Chrome",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           pathname: "/pageA",
           browser: "Chrome",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         )
       ])
@@ -759,21 +719,18 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
       conn: conn,
       site: site
     } do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           referrer_source: "Google",
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:event,
           name: "Signup",
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
-          domain: site.domain,
           referrer_source: "Twitter",
           timestamp: ~N[2021-01-01 00:25:00]
         )
@@ -796,26 +753,22 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
     end
 
     test "can breakdown by event:name when filtering by event:page", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           pathname: "/pageA",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           pathname: "/pageA",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:event,
           pathname: "/pageA",
           name: "Signup",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           pathname: "/pageB",
-          domain: site.domain,
           referrer_source: "Twitter",
           timestamp: ~N[2021-01-01 00:25:00]
         )
@@ -839,28 +792,24 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
     end
 
     test "can breakdown by event:page when filtering by event:name", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:event,
           name: "Signup",
           pathname: "/pageA",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:event,
           name: "Signup",
           pathname: "/pageA",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:event,
           name: "Signup",
           pathname: "/pageB",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           pathname: "/pageB",
-          domain: site.domain,
           referrer_source: "Twitter",
           timestamp: ~N[2021-01-01 00:25:00]
         )
@@ -911,33 +860,29 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
     end
 
     test "breakdown by custom event property", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:event,
           name: "Purchase",
           "meta.key": ["package"],
           "meta.value": ["business"],
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:event,
           name: "Purchase",
           "meta.key": ["package"],
           "meta.value": ["personal"],
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:event,
           name: "Purchase",
           "meta.key": ["package"],
           "meta.value": ["business"],
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
         build(:event,
           name: "Some other event",
           "meta.key": ["package"],
           "meta.value": ["business"],
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         )
       ])
@@ -960,45 +905,39 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
     end
 
     test "breakdown by custom event property, with (none)", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:event,
           name: "Purchase",
           "meta.key": ["cost"],
           "meta.value": ["16"],
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:event,
           name: "Purchase",
           "meta.key": ["cost"],
           "meta.value": ["16"],
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:event,
           name: "Purchase",
           "meta.key": ["cost"],
           "meta.value": ["16"],
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:event,
           name: "Purchase",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
         build(:event,
           name: "Purchase",
           "meta.key": ["cost"],
           "meta.value": ["14"],
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
         build(:event,
           name: "Purchase",
           "meta.key": ["cost"],
           "meta.value": ["14"],
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:26:00]
         )
       ])
@@ -1022,33 +961,29 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
     end
 
     test "breakdown by custom event property, limited", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:event,
           name: "Purchase",
           "meta.key": ["cost"],
           "meta.value": ["16"],
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:event,
           name: "Purchase",
           "meta.key": ["cost"],
           "meta.value": ["18"],
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
         build(:event,
           name: "Purchase",
           "meta.key": ["cost"],
           "meta.value": ["14"],
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
         build(:event,
           name: "Purchase",
           "meta.key": ["cost"],
           "meta.value": ["14"],
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:26:00]
         )
       ])
@@ -1072,40 +1007,35 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
     end
 
     test "breakdown by custom event property, paginated", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:event,
           name: "Purchase",
           "meta.key": ["cost"],
           "meta.value": ["16"],
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:event,
           name: "Purchase",
           "meta.key": ["cost"],
           "meta.value": ["16"],
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:event,
           name: "Purchase",
           "meta.key": ["cost"],
           "meta.value": ["18"],
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
         build(:event,
           name: "Purchase",
           "meta.key": ["cost"],
           "meta.value": ["14"],
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
         build(:event,
           name: "Purchase",
           "meta.key": ["cost"],
           "meta.value": ["14"],
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:26:00]
         )
       ])
@@ -1135,14 +1065,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
       insert(:goal, %{domain: site.domain, event_name: "Purchase"})
       insert(:goal, %{domain: site.domain, page_path: "/test"})
 
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00],
           pathname: "/test"
         ),
         build(:pageview,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:01],
           pathname: "/test",
           "meta.key": ["method"],
@@ -1150,14 +1078,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
         ),
         build(:event,
           name: "404",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:02],
           "meta.key": ["method"],
           "meta.value": ["HTTP"]
         ),
         build(:event,
           name: "Purchase",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:02],
           "meta.key": ["method"],
           "meta.value": ["HTTPS"]
@@ -1165,14 +1091,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
         build(:event,
           name: "404",
           timestamp: ~N[2021-01-01 00:00:03],
-          domain: site.domain,
           "meta.key": ["OS", "method"],
           "meta.value": ["Linux", "HTTP"]
         ),
         build(:event,
           name: "404",
           timestamp: ~N[2021-01-01 00:00:04],
-          domain: site.domain,
           "meta.key": ["version"],
           "meta.value": ["1"]
         )
@@ -1213,29 +1137,25 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
 
   describe "filtering" do
     test "event:page filter for breakdown by session props", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           pathname: "/ignore",
           browser: "Chrome",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           pathname: "/plausible.io",
           browser: "Chrome",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           pathname: "/plausible.io",
           browser: "Chrome",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
         build(:pageview,
           browser: "Safari",
           pathname: "/plausible.io",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         )
       ])
@@ -1409,25 +1329,21 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
     end
 
     test "IN filter for event:page", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           pathname: "/ignore",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           pathname: "/plausible.io",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           pathname: "/plausible.io",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           pathname: "/important-page",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         )
       ])
@@ -1450,29 +1366,25 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
     end
 
     test "IN filter for visit:browser", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           pathname: "/ignore",
           browser: "Firefox",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           pathname: "/plausible.io",
           browser: "Chrome",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           pathname: "/plausible.io",
           browser: "Safari",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           pathname: "/important-page",
           browser: "Safari",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         )
       ])
@@ -1495,25 +1407,21 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
     end
 
     test "IN filter for visit:entry_page", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           pathname: "/ignore",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           pathname: "/plausible.io",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           pathname: "/plausible.io",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           pathname: "/important-page",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         )
       ])
@@ -1537,25 +1445,21 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
     end
 
     test "IN filter for event:name", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:event,
           name: "Signup",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:event,
           name: "Signup",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:event,
           name: "Login",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:event,
           name: "Irrelevant",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         )
       ])
@@ -1603,10 +1507,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
 
   describe "pagination" do
     test "can limit results", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
-        build(:pageview, pathname: "/a", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
+        build(:pageview, pathname: "/a", timestamp: ~N[2021-01-01 00:00:00]),
-        build(:pageview, pathname: "/b", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
+        build(:pageview, pathname: "/b", timestamp: ~N[2021-01-01 00:25:00]),
-        build(:pageview, pathname: "/c", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+        build(:pageview, pathname: "/c", timestamp: ~N[2021-01-01 00:00:00])
       ])
 
       conn =
@@ -1623,11 +1527,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
     end
 
     test "does not repeat results", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
-        build(:pageview, %{domain: site.domain, "meta.key": ["item"], "meta.value": ["apple"]}),
+        build(:pageview, %{"meta.key": ["item"], "meta.value": ["apple"]}),
-        build(:pageview, %{domain: site.domain, "meta.key": ["item"], "meta.value": ["kiwi"]}),
+        build(:pageview, %{"meta.key": ["item"], "meta.value": ["kiwi"]}),
-        build(:pageview, %{domain: site.domain, "meta.key": ["item"], "meta.value": ["pineapple"]}),
+        build(:pageview, %{"meta.key": ["item"], "meta.value": ["pineapple"]}),
-        build(:pageview, %{domain: site.domain, "meta.key": ["item"], "meta.value": ["grapes"]})
+        build(:pageview, %{"meta.key": ["item"], "meta.value": ["grapes"]})
       ])
 
       params = %{
@@ -1693,10 +1597,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
     end
 
     test "can paginate results", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
-        build(:pageview, pathname: "/a", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
+        build(:pageview, pathname: "/a", timestamp: ~N[2021-01-01 00:00:00]),
-        build(:pageview, pathname: "/b", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
+        build(:pageview, pathname: "/b", timestamp: ~N[2021-01-01 00:25:00]),
-        build(:pageview, pathname: "/c", domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+        build(:pageview, pathname: "/c", timestamp: ~N[2021-01-01 00:00:00])
       ])
 
       conn =
@@ -1820,13 +1724,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
     end
 
     test "filter by custom event property", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:event,
           name: "Purchase",
           "meta.key": ["package"],
           "meta.value": ["business"],
           browser: "Chrome",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:event,
@@ -1834,7 +1737,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
           "meta.key": ["package"],
           "meta.value": ["business"],
           browser: "Safari",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:event,
@@ -1842,7 +1744,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
           "meta.key": ["package"],
           "meta.value": ["business"],
           browser: "Safari",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
         build(:event,
@@ -1850,7 +1751,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
           "meta.key": ["package"],
           "meta.value": ["personal"],
           browser: "IE",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         )
       ])
@@ -1873,23 +1773,20 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
     end
 
     test "all metrics for breakdown by event prop", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           user_id: 1,
           pathname: "/",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           user_id: 1,
           pathname: "/plausible.io",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:10:00]
         ),
-        build(:pageview, pathname: "/", domain: site.domain, timestamp: ~N[2021-01-01 00:25:00]),
+        build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),
         build(:pageview,
           pathname: "/plausible.io",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         )
       ])

test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs

@@ -261,9 +261,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
   end
 
   test "shows last 7 days of visitors", %{conn: conn, site: site} do
-    populate_stats([
+    populate_stats(site, [
-      build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
+      build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
-      build(:pageview, domain: site.domain, timestamp: ~N[2021-01-07 23:59:00])
+      build(:pageview, timestamp: ~N[2021-01-07 23:59:00])
     ])
 
     conn =
@@ -287,10 +287,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
   end
 
   test "shows last 6 months of visitors", %{conn: conn, site: site} do
-    populate_stats([
+    populate_stats(site, [
-      build(:pageview, domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]),
+      build(:pageview, timestamp: ~N[2020-12-31 00:00:00]),
-      build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
+      build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
-      build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+      build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
     ])
 
     conn =
@@ -313,11 +313,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
   end
 
   test "shows last 12 months of visitors", %{conn: conn, site: site} do
-    populate_stats([
+    populate_stats(site, [
-      build(:pageview, domain: site.domain, timestamp: ~N[2020-02-01 00:00:00]),
+      build(:pageview, timestamp: ~N[2020-02-01 00:00:00]),
-      build(:pageview, domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]),
+      build(:pageview, timestamp: ~N[2020-12-31 00:00:00]),
-      build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
+      build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
-      build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+      build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
     ])
 
     conn =
@@ -346,11 +346,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
   end
 
   test "shows last 12 months of visitors with interval daily", %{conn: conn, site: site} do
-    populate_stats([
+    populate_stats(site, [
-      build(:pageview, domain: site.domain, timestamp: ~N[2020-02-01 00:00:00]),
+      build(:pageview, timestamp: ~N[2020-02-01 00:00:00]),
-      build(:pageview, domain: site.domain, timestamp: ~N[2020-12-31 00:00:00]),
+      build(:pageview, timestamp: ~N[2020-12-31 00:00:00]),
-      build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
+      build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
-      build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+      build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
     ])
 
     conn =
@@ -365,10 +365,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
   end
 
   test "shows a custom range with daily interval", %{conn: conn, site: site} do
-    populate_stats([
+    populate_stats(site, [
-      build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
+      build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
-      build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
+      build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
-      build(:pageview, domain: site.domain, timestamp: ~N[2021-01-02 00:00:00])
+      build(:pageview, timestamp: ~N[2021-01-02 00:00:00])
     ])
 
     conn =
@@ -387,11 +387,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
   end
 
   test "shows a custom range with monthly interval", %{conn: conn, site: site} do
-    populate_stats([
+    populate_stats(site, [
-      build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2020-12-01 00:00:00]),
+      build(:pageview, user_id: @user_id, timestamp: ~N[2020-12-01 00:00:00]),
-      build(:pageview, user_id: @user_id, domain: site.domain, timestamp: ~N[2020-12-01 00:05:00]),
+      build(:pageview, user_id: @user_id, timestamp: ~N[2020-12-01 00:05:00]),
-      build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
+      build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
-      build(:pageview, domain: site.domain, timestamp: ~N[2021-01-02 00:00:00])
+      build(:pageview, timestamp: ~N[2021-01-02 00:00:00])
     ])
 
     conn =
@@ -425,13 +425,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
 
   describe "filters" do
     test "can filter by source", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           referrer_source: "Google",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
       ])
 
       conn =
@@ -447,11 +446,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
     end
 
     test "can filter by no source/referrer", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
         build(:pageview,
           referrer_source: "Google",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         )
       ])
@@ -469,13 +467,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
     end
 
     test "can filter by referrer", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           referrer: "https://facebook.com",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
       ])
 
       conn =
@@ -491,13 +488,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
     end
 
     test "can filter by utm_medium", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           utm_medium: "social",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
       ])
 
       conn =
@@ -513,13 +509,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
     end
 
     test "can filter by utm_source", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           utm_source: "Twitter",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
       ])
 
       conn =
@@ -535,13 +530,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
     end
 
     test "can filter by utm_campaign", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           utm_campaign: "profile",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
       ])
 
       conn =
@@ -557,13 +551,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
     end
 
     test "can filter by device type", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           screen_size: "Desktop",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
       ])
 
       conn =
@@ -579,20 +572,18 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
     end
 
     test "can filter by browser", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           browser: "Chrome",
           browser_version: "56.1",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           browser: "Chrome",
           browser_version: "55",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
       ])
 
       conn =
@@ -608,26 +599,23 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
     end
 
     test "can filter by operating system", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           operating_system: "Mac",
           operating_system_version: "10.5",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           operating_system: "Something else",
           operating_system_version: "10.5",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           operating_system: "Mac",
           operating_system_version: "10.4",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
       ])
 
       conn =
@@ -643,22 +631,20 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
     end
 
     test "can filter by country", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           user_id: @user_id,
           country_code: "EE",
           operating_system_version: "10.5",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           user_id: @user_id,
           country_code: "EE",
           operating_system_version: "10.5",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:15:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
       ])
 
       conn =
@@ -685,26 +671,22 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
       conn: conn,
       site: site
     } do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           pathname: "/hello",
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:05:00]
         ),
         build(:pageview,
           pathname: "/hello",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 05:00:00]
         ),
         build(:pageview,
           pathname: "/goobye",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         )
       ])
@@ -732,13 +714,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
     end
 
     test "can filter by event:name", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:event,
           name: "Signup",
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00])
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00])
       ])
 
       conn =
@@ -754,33 +735,29 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
     end
 
     test "filter by custom event property", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:event,
           name: "Purchase",
           "meta.key": ["package"],
           "meta.value": ["business"],
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:event,
           name: "Purchase",
           "meta.key": ["package"],
           "meta.value": ["business"],
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:event,
           name: "Purchase",
           "meta.key": ["package"],
           "meta.value": ["personal"],
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:25:00]
         ),
         build(:event,
           name: "Purchase",
           "meta.key": ["package"],
           "meta.value": ["business"],
-          domain: site.domain,
           timestamp: ~N[2021-01-02 00:25:00]
         )
       ])
@@ -801,19 +778,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
 
   describe "metrics" do
     test "shows pageviews,visits,views_per_visit for last 7d", %{conn: conn, site: site} do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:05:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-01 00:00:00]),
+        build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-07 23:59:00])
+        build(:pageview, timestamp: ~N[2021-01-07 23:59:00])
       ])
 
       conn =
@@ -876,30 +851,26 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
       conn: conn,
       site: site
     } do
-      populate_stats([
+      populate_stats(site, [
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:00:00]
         ),
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-01 00:05:00]
         ),
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-03 00:00:00]
         ),
         build(:pageview,
           user_id: @user_id,
-          domain: site.domain,
           timestamp: ~N[2021-01-03 00:01:00]
         ),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-03 00:00:00]),
+        build(:pageview, timestamp: ~N[2021-01-03 00:00:00]),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-03 00:00:00]),
+        build(:pageview, timestamp: ~N[2021-01-03 00:00:00]),
-        build(:pageview, domain: site.domain, timestamp: ~N[2021-01-07 23:59:00])
+        build(:pageview, timestamp: ~N[2021-01-07 23:59:00])
       ])
 
       conn =

test/plausible_web/controllers/api/internal_controller_test.exs

@@ -14,7 +14,7 @@ defmodule PlausibleWeb.Api.InternalControllerTest do
 
     test "is READY when site has at least 1 pageview", %{conn: conn, user: user} do
       site = insert(:site, members: [user])
-      Plausible.TestUtils.create_pageviews([%{domain: site.domain}])
+      Plausible.TestUtils.create_pageviews([%{site: site}])
 
       conn = get(conn, "/api/#{site.domain}/status")
 

test/plausible_web/controllers/api/stats_controller/conversions_test.exs

@@ -653,6 +653,41 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
       insert(:goal, %{domain: site.domain, page_path: "/*"})
       insert(:goal, %{domain: site.domain, page_path: "/**"})
 
+      populate_stats(site, [
+        build(:pageview,
+          pathname: "/hum",
+          timestamp: ~N[2019-07-01 23:00:00]
+        ),
+        build(:pageview,
+          pathname: "/register",
+          timestamp: ~N[2019-07-01 23:00:00]
+        ),
+        build(:pageview,
+          pathname: "/reg",
+          timestamp: ~N[2019-07-01 23:00:00]
+        ),
+        build(:pageview,
+          pathname: "/billing/success",
+          timestamp: ~N[2019-07-01 23:00:00]
+        ),
+        build(:pageview,
+          pathname: "/billing/upgrade/success",
+          timestamp: ~N[2019-07-01 23:00:00]
+        ),
+        build(:pageview,
+          pathname: "/signup/new",
+          timestamp: ~N[2019-07-01 23:00:00]
+        ),
+        build(:pageview,
+          pathname: "/signup/new/2",
+          timestamp: ~N[2019-07-01 23:00:00]
+        ),
+        build(:pageview,
+          pathname: "/signup/new/3",
+          timestamp: ~N[2019-07-01 23:00:00]
+        )
+      ])
+
       conn =
         get(
           conn,

test/plausible_web/controllers/api/stats_controller/current_visitors_test.exs

@@ -5,6 +5,11 @@ defmodule PlausibleWeb.Api.StatsController.CurrentVisitorsTest do
     setup [:create_user, :log_in, :create_site]
 
     test "returns unique users in the last 5 minutes", %{conn: conn, site: site} do
+      populate_stats(site, [
+        build(:pageview, user_id: 123),
+        build(:pageview, user_id: 456)
+      ])
+
       conn = get(conn, "/api/stats/#{site.domain}/current-visitors")
 
       assert json_response(conn, 200) == 2

test/plausible_web/controllers/api/stats_controller/suggestions_test.exs

@@ -5,23 +5,37 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
     setup [:create_user, :log_in, :create_site]
 
     test "returns suggestions for pages without a query", %{conn: conn, site: site} do
+      populate_stats(site, [
+        build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/"),
+        build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/register"),
+        build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/contact"),
+        build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/irrelevant")
+      ])
+
       conn = get(conn, "/api/stats/#{site.domain}/suggestions/page?period=month&date=2019-01-01")
 
       assert json_response(conn, 200) == [
                %{"label" => "/", "value" => "/"},
-               %{"label" => "/register", "value" => "/register"},
                %{"label" => "/contact", "value" => "/contact"},
-               %{"label" => "/irrelevant", "value" => "/irrelevant"}
+               %{"label" => "/irrelevant", "value" => "/irrelevant"},
+               %{"label" => "/register", "value" => "/register"}
              ]
     end
 
     test "returns suggestions for pages with a query", %{conn: conn, site: site} do
+      populate_stats(site, [
+        build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/"),
+        build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/register"),
+        build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/contact"),
+        build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/irrelevant")
+      ])
+
       conn =
         get(conn, "/api/stats/#{site.domain}/suggestions/page?period=month&date=2019-01-01&q=re")
 
       assert json_response(conn, 200) == [
-               %{"label" => "/register", "value" => "/register"},
+               %{"label" => "/irrelevant", "value" => "/irrelevant"},
-               %{"label" => "/irrelevant", "value" => "/irrelevant"}
+               %{"label" => "/register", "value" => "/register"}
              ]
     end
 
@@ -42,16 +56,25 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
     end
 
     test "returns suggestions for sources", %{conn: conn, site: site} do
+      populate_stats(site, [
+        build(:pageview, timestamp: ~N[2019-01-01 23:00:00], referrer_source: "Bing"),
+        build(:pageview, timestamp: ~N[2019-01-01 23:00:00], referrer_source: "10words")
+      ])
+
       conn =
         get(conn, "/api/stats/#{site.domain}/suggestions/source?period=month&date=2019-01-01")
 
       assert json_response(conn, 200) == [
-               %{"label" => "10words", "value" => "10words"},
+               %{"label" => "Bing", "value" => "Bing"},
-               %{"label" => "Bing", "value" => "Bing"}
+               %{"label" => "10words", "value" => "10words"}
              ]
     end
 
     test "returns suggestions for countries", %{conn: conn, site: site} do
+      populate_stats(site, [
+        build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/", country_code: "US")
+      ])
+
       conn =
         get(
           conn,
@@ -106,6 +129,10 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
     end
 
     test "returns suggestions for screen sizes", %{conn: conn, site: site} do
+      populate_stats(site, [
+        build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/", screen_size: "Desktop")
+      ])
+
       conn =
         get(conn, "/api/stats/#{site.domain}/suggestions/screen?period=month&date=2019-01-01")
 
@@ -113,6 +140,10 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
     end
 
     test "returns suggestions for browsers", %{conn: conn, site: site} do
+      populate_stats(site, [
+        build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/", browser: "Chrome")
+      ])
+
       conn =
         get(conn, "/api/stats/#{site.domain}/suggestions/browser?period=month&date=2019-01-01")
 
@@ -122,6 +153,14 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
     test "returns suggestions for browser versions", %{conn: conn, site: site} do
       filters = Jason.encode!(%{browser: "Chrome"})
 
+      populate_stats(site, [
+        build(:pageview,
+          timestamp: ~N[2019-01-01 00:00:00],
+          browser: "Chrome",
+          browser_version: "78.0"
+        )
+      ])
+
       conn =
         get(
           conn,
@@ -132,6 +171,10 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
     end
 
     test "returns suggestions for OS", %{conn: conn, site: site} do
+      populate_stats(site, [
+        build(:pageview, timestamp: ~N[2019-01-01 00:00:00], operating_system: "Mac")
+      ])
+
       conn = get(conn, "/api/stats/#{site.domain}/suggestions/os?period=month&date=2019-01-01")
 
       assert json_response(conn, 200) == [%{"value" => "Mac", "label" => "Mac"}]
@@ -140,6 +183,14 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
     test "returns suggestions for OS versions", %{conn: conn, site: site} do
       filters = Jason.encode!(%{os: "Mac"})
 
+      populate_stats(site, [
+        build(:pageview,
+          timestamp: ~N[2019-01-01 00:00:00],
+          operating_system: "Mac",
+          operating_system_version: "10.15"
+        )
+      ])
+
       conn =
         get(
           conn,
@@ -162,6 +213,14 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
     end
 
     test "returns suggestions for referrers", %{conn: conn, site: site} do
+      populate_stats(site, [
+        build(:pageview,
+          timestamp: ~N[2019-01-01 23:00:00],
+          pathname: "/",
+          referrer: "10words.com/page1"
+        )
+      ])
+
       conn =
         get(conn, "/api/stats/#{site.domain}/suggestions/referrer?period=month&date=2019-01-01")
 

test/plausible_web/controllers/site_controller_test.exs

@@ -41,10 +41,12 @@ defmodule PlausibleWeb.SiteControllerTest do
     end
 
     test "lists all of your sites with last 24h visitors", %{conn: conn, user: user} do
-      insert(:site, members: [user], domain: "test-site.com")
+      site = insert(:site, members: [user])
+
+      populate_stats(site, [build(:pageview), build(:pageview), build(:pageview)])
       conn = get(conn, "/sites")
 
-      assert html_response(conn, 200) =~ "test-site.com"
+      assert html_response(conn, 200) =~ site.domain
       assert html_response(conn, 200) =~ "<b>3</b> visitors in last 24h"
     end
 

test/plausible_web/controllers/stats_controller_test.exs

@@ -4,9 +4,10 @@ defmodule PlausibleWeb.StatsControllerTest do
 
   describe "GET /:website - anonymous user" do
     test "public site - shows site stats", %{conn: conn} do
-      insert(:site, domain: "public-site.io", public: true)
+      site = insert(:site, public: true)
+      populate_stats(site, [build(:pageview)])
 
-      conn = get(conn, "/public-site.io")
+      conn = get(conn, "/#{site.domain}")
       assert html_response(conn, 200) =~ "stats-react-container"
     end
 
@@ -27,6 +28,7 @@ defmodule PlausibleWeb.StatsControllerTest do
     setup [:create_user, :log_in, :create_site]
 
     test "can view stats of a website I've created", %{conn: conn, site: site} do
+      populate_stats(site, [build(:pageview)])
       conn = get(conn, "/" <> site.domain)
       assert html_response(conn, 200) =~ "stats-react-container"
     end

test/support/clickhouse_setup.ex

@@ -1,337 +0,0 @@
-defmodule Plausible.Test.ClickhouseSetup do
-  @conversion_1_session_id 123
-  @conversion_2_session_id 234
-
-  def run() do
-    Plausible.TestUtils.create_events([
-      %{
-        name: "pageview",
-        domain: "test-site.com",
-        pathname: "/",
-        country_code: "EE",
-        browser: "Chrome",
-        browser_version: "78.0",
-        operating_system: "Mac",
-        operating_system_version: "10.15",
-        screen_size: "Desktop",
-        referrer_source: "10words",
-        referrer: "10words.com/page1",
-        timestamp: ~N[2019-01-01 00:00:00],
-        session_id: @conversion_1_session_id
-      },
-      %{
-        name: "pageview",
-        domain: "test-site.com",
-        pathname: "/",
-        country_code: "EE",
-        browser: "Chrome",
-        browser_version: "78.0",
-        operating_system: "Mac",
-        operating_system_version: "10.15",
-        screen_size: "Desktop",
-        referrer_source: "10words",
-        referrer: "10words.com/page2",
-        timestamp: ~N[2019-01-01 00:00:00]
-      },
-      %{
-        name: "pageview",
-        domain: "test-site.com",
-        pathname: "/",
-        country_code: "US",
-        browser: "Chrome",
-        browser_version: "79.0",
-        operating_system: "Mac",
-        operating_system_version: "10.16",
-        screen_size: "Laptop",
-        referrer_source: "",
-        referrer: "",
-        timestamp: ~N[2019-01-01 00:00:00]
-      },
-      %{
-        name: "pageview",
-        domain: "test-site.com",
-        pathname: "/contact",
-        country_code: "GB",
-        browser: "Firefox",
-        operating_system: "Android",
-        screen_size: "Mobile",
-        referrer_source: "Bing",
-        timestamp: ~N[2019-01-01 00:00:00]
-      },
-      %{name: "pageview", domain: "test-site.com", timestamp: ~N[2019-01-31 00:00:00]},
-      %{
-        name: "Signup",
-        domain: "test-site.com",
-        session_id: @conversion_1_session_id,
-        timestamp: ~N[2019-01-01 01:00:00],
-        "meta.key": ["variant"],
-        "meta.value": ["A"]
-      },
-      %{
-        name: "Signup",
-        domain: "test-site.com",
-        session_id: @conversion_1_session_id,
-        timestamp: ~N[2019-01-01 02:00:00],
-        "meta.key": ["variant"],
-        "meta.value": ["B"]
-      },
-      %{
-        name: "Signup",
-        domain: "test-site.com",
-        session_id: @conversion_2_session_id,
-        timestamp: ~N[2019-01-01 02:00:00],
-        "meta.key": ["variant"],
-        "meta.value": ["B"]
-      },
-      %{
-        name: "pageview",
-        pathname: "/register",
-        domain: "test-site.com",
-        session_id: @conversion_1_session_id,
-        timestamp: ~N[2019-01-01 23:00:00]
-      },
-      %{
-        name: "pageview",
-        pathname: "/register",
-        domain: "test-site.com",
-        session_id: @conversion_2_session_id,
-        timestamp: ~N[2019-01-01 23:00:00]
-      },
-      %{
-        name: "pageview",
-        pathname: "/irrelevant",
-        domain: "test-site.com",
-        session_id: @conversion_1_session_id,
-        timestamp: ~N[2019-01-01 23:00:01]
-      },
-      %{
-        name: "pageview",
-        domain: "test-site.com",
-        referrer_source: "Google",
-        timestamp: ~N[2019-02-01 01:00:00]
-      },
-      %{
-        name: "pageview",
-        domain: "test-site.com",
-        referrer_source: "Google",
-        timestamp: ~N[2019-02-01 02:00:00]
-      },
-      %{
-        name: "pageview",
-        domain: "test-site.com",
-        referrer: "t.co/some-link",
-        referrer_source: "Twitter",
-        timestamp: ~N[2019-03-01 01:00:00]
-      },
-      %{
-        name: "pageview",
-        domain: "test-site.com",
-        referrer: "t.co/some-link",
-        referrer_source: "Twitter",
-        timestamp: ~N[2019-03-01 01:00:00]
-      },
-      %{
-        name: "pageview",
-        domain: "test-site.com",
-        referrer: "t.co/nonexistent-link",
-        referrer_source: "Twitter",
-        timestamp: ~N[2019-03-01 02:00:00]
-      },
-      %{name: "pageview", domain: "test-site.com"},
-      %{
-        name: "pageview",
-        domain: "test-site.com",
-        timestamp: Timex.now() |> Timex.shift(minutes: -3)
-      },
-      %{
-        name: "pageview",
-        domain: "test-site.com",
-        timestamp: Timex.now() |> Timex.shift(minutes: -6)
-      },
-      %{name: "pageview", domain: "tz-test.com", timestamp: ~N[2019-01-01 00:00:00]},
-      %{name: "pageview", domain: "public-site.io"},
-      %{
-        name: "pageview",
-        domain: "fetch-tweets-test.com",
-        referrer: "t.co/a-link",
-        referrer_source: "Twitter"
-      },
-      %{
-        name: "pageview",
-        domain: "fetch-tweets-test.com",
-        referrer: "t.co/b-link",
-        referrer_source: "Twitter",
-        timestamp: Timex.now() |> Timex.shift(days: -5)
-      },
-      %{
-        name: "pageview",
-        pathname: "/register",
-        domain: "test-site.com",
-        timestamp: ~N[2019-07-01 23:00:00]
-      },
-      %{
-        name: "pageview",
-        pathname: "/signup/new",
-        domain: "test-site.com",
-        timestamp: ~N[2019-07-01 23:00:00]
-      },
-      %{
-        name: "pageview",
-        pathname: "/billing/success",
-        domain: "test-site.com",
-        timestamp: ~N[2019-07-01 23:00:00]
-      },
-      %{
-        name: "pageview",
-        pathname: "/",
-        domain: "test-site.com",
-        timestamp: ~N[2019-07-01 23:00:00]
-      },
-      %{
-        name: "pageview",
-        pathname: "/reg",
-        domain: "test-site.com",
-        timestamp: ~N[2019-07-01 23:00:00]
-      },
-      %{
-        name: "pageview",
-        pathname: "/signup/new/2",
-        domain: "test-site.com",
-        timestamp: ~N[2019-07-01 23:00:00]
-      },
-      %{
-        name: "pageview",
-        pathname: "/signup/new/3",
-        domain: "test-site.com",
-        timestamp: ~N[2019-07-01 23:00:00]
-      },
-      %{
-        name: "pageview",
-        pathname: "/billing/upgrade/success",
-        domain: "test-site.com",
-        timestamp: ~N[2019-07-01 23:00:00]
-      }
-    ])
-
-    Plausible.TestUtils.create_sessions([
-      %{
-        domain: "test-site.com",
-        entry_page: "/",
-        exit_page: "/",
-        referrer_source: "10words",
-        referrer: "10words.com/page1",
-        utm_medium: "listing",
-        country_code: "US",
-        screen_size: "Desktop",
-        browser: "Chrome",
-        browser_version: "78.0",
-        operating_system: "Mac",
-        operating_system_version: "10.15",
-        session_id: @conversion_1_session_id,
-        is_bounce: true,
-        duration: 100,
-        start: ~N[2019-01-01 02:00:00],
-        timestamp: ~N[2019-01-01 02:00:00]
-      },
-      %{
-        domain: "test-site.com",
-        entry_page: "/",
-        exit_page: "/",
-        referrer_source: "10words",
-        referrer: "10words.com/page1",
-        utm_medium: "listing",
-        session_id: @conversion_2_session_id,
-        is_bounce: false,
-        duration: 0,
-        start: ~N[2019-01-01 02:00:00],
-        timestamp: ~N[2019-01-01 02:00:00]
-      },
-      %{
-        domain: "test-site.com",
-        entry_page: "/",
-        exit_page: "/",
-        referrer_source: "Bing",
-        referrer: "",
-        utm_medium: "search",
-        is_bounce: false,
-        duration: 100,
-        start: ~N[2019-01-01 03:00:00],
-        timestamp: ~N[2019-01-01 03:00:00]
-      },
-      %{
-        domain: "test-site.com",
-        entry_page: "/",
-        exit_page: "/",
-        referrer_source: "Google",
-        referrer: "",
-        is_bounce: false,
-        start: ~N[2019-02-01 01:00:00],
-        timestamp: ~N[2019-02-01 01:00:00]
-      },
-      %{
-        domain: "test-site.com",
-        entry_page: "/",
-        exit_page: "/",
-        referrer_source: "Google",
-        referrer: "",
-        is_bounce: false,
-        start: ~N[2019-02-01 02:00:00],
-        timestamp: ~N[2019-02-01 02:00:00]
-      },
-      %{
-        domain: "test-site.com",
-        entry_page: "/",
-        exit_page: "/",
-        referrer: "t.co/some-link",
-        referrer_source: "Twitter",
-        start: ~N[2019-03-01 01:00:00],
-        timestamp: ~N[2019-03-01 01:00:00]
-      },
-      %{
-        domain: "test-site.com",
-        entry_page: "/",
-        exit_page: "/",
-        referrer: "t.co/some-link",
-        referrer_source: "Twitter",
-        start: ~N[2019-03-01 01:00:00],
-        timestamp: ~N[2019-03-01 01:00:00]
-      },
-      %{
-        domain: "test-site.com",
-        entry_page: "/",
-        exit_page: "/",
-        referrer: "t.co/nonexistent-link",
-        referrer_source: "Twitter",
-        start: ~N[2019-03-01 02:00:00],
-        timestamp: ~N[2019-03-01 02:00:00]
-      },
-      %{
-        domain: "test-site.com",
-        entry_page: "/",
-        exit_page: "/",
-        referrer_source: "Bing",
-        referrer: "bing.com",
-        start: Timex.now() |> Timex.shift(minutes: -1),
-        timestamp: Timex.now() |> Timex.shift(minutes: -1)
-      },
-      %{
-        domain: "test-site.com",
-        entry_page: "/",
-        exit_page: "/exit",
-        referrer_source: "10words",
-        referrer: "10words.com",
-        start: Timex.now() |> Timex.shift(minutes: -2),
-        timestamp: Timex.now() |> Timex.shift(minutes: -2)
-      },
-      %{
-        domain: "test-site.com",
-        entry_page: "/",
-        exit_page: "/exit",
-        referrer_source: "10words",
-        referrer: "10words.com",
-        start: Timex.now() |> Timex.shift(minutes: -3),
-        timestamp: Timex.now() |> Timex.shift(minutes: -3)
-      }
-    ])
-  end
-end

test/support/factory.ex

@@ -38,6 +38,21 @@ defmodule Plausible.Factory do
   def ch_session_factory do
     hostname = sequence(:domain, &"example-#{&1}.com")
 
+    if Plausible.v2?() do
+      %Plausible.ClickhouseSessionV2{
+        sign: 1,
+        session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
+        user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
+        hostname: hostname,
+        site_id: Enum.random(1000..10_000),
+        entry_page: "/",
+        pageviews: 1,
+        events: 1,
+        start: Timex.now(),
+        timestamp: Timex.now(),
+        is_bounce: false
+      }
+    else
       %Plausible.ClickhouseSession{
         sign: 1,
         session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
@@ -52,6 +67,7 @@ defmodule Plausible.Factory do
         is_bounce: false
       }
     end
+  end
 
   def pageview_factory do
     struct!(
@@ -65,6 +81,16 @@ defmodule Plausible.Factory do
   def event_factory do
     hostname = sequence(:domain, &"example-#{&1}.com")
 
+    if Plausible.v2?() do
+      %Plausible.ClickhouseEventV2{
+        hostname: hostname,
+        site_id: Enum.random(1000..10_000),
+        pathname: "/",
+        timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second),
+        user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
+        session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate())
+      }
+    else
       %Plausible.ClickhouseEvent{
         hostname: hostname,
         domain: hostname,
@@ -74,6 +100,7 @@ defmodule Plausible.Factory do
         session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate())
       }
     end
+  end
 
   def goal_factory do
     %Plausible.Goal{}

test/support/test_utils.ex

@@ -37,7 +37,6 @@ defmodule Plausible.TestUtils do
   def create_site(%{user: user}) do
     site =
       Factory.insert(:site,
-        domain: "test-site.com",
         members: [user]
       )
 
@@ -73,30 +72,63 @@ defmodule Plausible.TestUtils do
   def create_pageviews(pageviews) do
     pageviews =
       Enum.map(pageviews, fn pageview ->
+        pageview =
+          if Plausible.v2?() do
+            pageview
+            |> Map.delete(:site)
+            |> Map.put(:site_id, pageview.site.id)
+          else
+            pageview
+            |> Map.delete(:site)
+            |> Map.put(:domain, pageview.site.domain)
+          end
+
         Factory.build(:pageview, pageview)
         |> Map.from_struct()
         |> Map.delete(:__meta__)
         |> update_in([:timestamp], &to_naive_truncate/1)
       end)
 
+    if Plausible.v2?() do
+      Plausible.IngestRepo.insert_all(Plausible.ClickhouseEventV2, pageviews)
+    else
       Plausible.IngestRepo.insert_all(Plausible.ClickhouseEvent, pageviews)
     end
+  end
 
   def create_events(events) do
     events =
       Enum.map(events, fn event ->
+        event =
+          if Plausible.v2?() do
+            Map.delete(event, :domain)
+          else
+            Map.delete(event, :site_id)
+          end
+
         Factory.build(:event, event)
         |> Map.from_struct()
         |> Map.delete(:__meta__)
         |> update_in([:timestamp], &to_naive_truncate/1)
       end)
 
+    if Plausible.v2?() do
+      Plausible.IngestRepo.insert_all(Plausible.ClickhouseEventV2, events)
+    else
       Plausible.IngestRepo.insert_all(Plausible.ClickhouseEvent, events)
     end
+  end
 
   def create_sessions(sessions) do
     sessions =
       Enum.map(sessions, fn session ->
+        session =
+          if Plausible.v2?() do
+            Map.delete(session, :domain)
+          else
+            Map.delete(session, :site_id)
+          end
+
         Factory.build(:ch_session, session)
         |> Map.from_struct()
         |> Map.delete(:__meta__)
@@ -104,8 +136,12 @@ defmodule Plausible.TestUtils do
         |> update_in([:start], &to_naive_truncate/1)
       end)
 
+    if Plausible.v2?() do
+      Plausible.IngestRepo.insert_all(Plausible.ClickhouseSessionV2, sessions)
+    else
       Plausible.IngestRepo.insert_all(Plausible.ClickhouseSession, sessions)
     end
+  end
 
   def log_in(%{user: user, conn: conn}) do
     conn =
@@ -134,6 +170,9 @@ defmodule Plausible.TestUtils do
   def populate_stats(site, events) do
     Enum.map(events, fn event ->
       case event do
+        %Plausible.ClickhouseEventV2{} ->
+          Map.put(event, :site_id, site.id)
+
         %Plausible.ClickhouseEvent{} ->
           Map.put(event, :domain, site.domain)
 
@@ -158,6 +197,9 @@ defmodule Plausible.TestUtils do
       end)
       |> Enum.split_with(fn event ->
         case event do
+          %Plausible.ClickhouseEventV2{} ->
+            true
+
           %Plausible.ClickhouseEvent{} ->
             true
 
@@ -174,11 +216,22 @@ defmodule Plausible.TestUtils do
     sessions =
       Enum.reduce(events, %{}, fn event, sessions ->
         session_id = Plausible.Session.CacheStore.on_event(event, nil)
+
+        if Plausible.v2?() do
+          Map.put(sessions, {event.site_id, event.user_id}, session_id)
+        else
           Map.put(sessions, {event.domain, event.user_id}, session_id)
+        end
       end)
 
     Enum.each(events, fn event ->
-      event = Map.put(event, :session_id, sessions[{event.domain, event.user_id}])
+      event =
+        if Plausible.v2?() do
+          Map.put(event, :session_id, sessions[{event.site_id, event.user_id}])
+        else
+          Map.put(event, :session_id, sessions[{event.domain, event.user_id}])
+        end
+
       Plausible.Event.WriteBuffer.insert(event)
     end)
 

test/test_helper.exs

@@ -1,7 +1,14 @@
 {:ok, _} = Application.ensure_all_started(:ex_machina)
-Plausible.Test.ClickhouseSetup.run()
 Mox.defmock(Plausible.HTTPClient.Mock, for: Plausible.HTTPClient.Interface)
 FunWithFlags.enable(:visits_metric)
 ExUnit.start(exclude: :slow)
 Application.ensure_all_started(:double)
 Ecto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual)
+
+if Plausible.v2?() do
+  IO.puts("Running tests against v2 schema")
+else
+  IO.puts(
+    "Running tests against v1 schema. Use: `V2_MIGRATION_DONE=1 mix test` for secondary run."
+  )
+end

test/workers/send_check_stats_emails_test.exs

@@ -25,6 +25,8 @@ defmodule Plausible.Workers.SendCheckStatsEmailsTest do
   test "does not send an email if the user has configured a weekly report" do
     user = insert(:user, inserted_at: days_ago(9), last_seen: days_ago(7))
     site = insert(:site, domain: "test-site.com", members: [user])
+
+    populate_stats(site, [build(:pageview)])
     insert(:weekly_report, site: site, recipients: ["user@email.com"])
 
     perform_job(SendCheckStatsEmails, %{})
@@ -34,7 +36,8 @@ defmodule Plausible.Workers.SendCheckStatsEmailsTest do
 
   test "sends an email after a week of signup if the user hasn't logged in" do
     user = insert(:user, inserted_at: days_ago(8), last_seen: days_ago(8))
-    insert(:site, domain: "test-site.com", members: [user])
+    site = insert(:site, domain: "test-site.com", members: [user])
+    populate_stats(site, [build(:pageview)])
 
     perform_job(SendCheckStatsEmails, %{})
 

test/workers/send_email_report_test.exs

@@ -39,17 +39,17 @@ defmodule Plausible.Workers.SendEmailReportTest do
 
       create_pageviews([
         # Sunday before last, not counted
-        %{domain: site.domain, timestamp: Timezone.convert(sunday_before_last, "UTC")},
+        %{site: site, timestamp: Timezone.convert(sunday_before_last, "UTC")},
         # Sunday before last, not counted
-        %{domain: site.domain, timestamp: Timezone.convert(sunday_before_last, "UTC")},
+        %{site: site, timestamp: Timezone.convert(sunday_before_last, "UTC")},
         # Last monday, counted
-        %{domain: site.domain, timestamp: Timezone.convert(last_monday, "UTC")},
+        %{site: site, timestamp: Timezone.convert(last_monday, "UTC")},
         # Last sunday, counted
-        %{domain: site.domain, timestamp: Timezone.convert(last_sunday, "UTC")},
+        %{site: site, timestamp: Timezone.convert(last_sunday, "UTC")},
         # This monday, not counted
-        %{domain: site.domain, timestamp: Timezone.convert(this_monday, "UTC")},
+        %{site: site, timestamp: Timezone.convert(this_monday, "UTC")},
         # This monday, not counted
-        %{domain: site.domain, timestamp: Timezone.convert(this_monday, "UTC")}
+        %{site: site, timestamp: Timezone.convert(this_monday, "UTC")}
       ])
 
       perform_job(SendEmailReport, %{"site_id" => site.id, "interval" => "weekly"})

test/workers/send_site_setup_emails_test.exs

@@ -41,7 +41,9 @@ defmodule Plausible.Workers.SendSiteSetupEmailsTest do
     test "sends the setup completed email as soon as possible" do
       user = insert(:user)
 
-      insert(:site, members: [user], domain: "test-site.com")
+      site = insert(:site, members: [user])
+
+      populate_stats(site, [build(:pageview)])
 
       perform_job(SendSiteSetupEmails, %{})
 
@@ -62,7 +64,7 @@ defmodule Plausible.Workers.SendSiteSetupEmailsTest do
         subject: "Your Plausible setup: Waiting for the first page views"
       )
 
-      create_pageviews([%{domain: site.domain}])
+      create_pageviews([%{site: site}])
       perform_job(SendSiteSetupEmails, %{})
 
       assert_email_delivered_with(

test/workers/send_trial_notifications_test.exs

@@ -44,7 +44,7 @@ defmodule Plausible.Workers.SendTrialNotificationsTest do
         ]
       )
 
-    populate_stats(site, [build(:pageview, domain: site.domain)])
+    populate_stats(site, [build(:pageview)])
 
     perform_job(SendTrialNotifications, %{})
 
@@ -55,7 +55,7 @@ defmodule Plausible.Workers.SendTrialNotificationsTest do
     test "sends a reminder 7 days before trial ends (16 days after user signed up)" do
       user = insert(:user, trial_expiry_date: Timex.now() |> Timex.shift(days: 7))
       site = insert(:site, members: [user])
-      populate_stats(site, [build(:pageview, domain: site.domain)])
+      populate_stats(site, [build(:pageview)])
 
       perform_job(SendTrialNotifications, %{})
 
@@ -67,9 +67,9 @@ defmodule Plausible.Workers.SendTrialNotificationsTest do
       site = insert(:site, members: [user])
 
       populate_stats(site, [
-        build(:pageview, domain: site.domain),
+        build(:pageview),
-        build(:pageview, domain: site.domain),
+        build(:pageview),
-        build(:pageview, domain: site.domain)
+        build(:pageview)
       ])
 
       perform_job(SendTrialNotifications, %{})
@@ -82,9 +82,9 @@ defmodule Plausible.Workers.SendTrialNotificationsTest do
       site = insert(:site, members: [user])
 
       populate_stats(site, [
-        build(:pageview, domain: site.domain),
+        build(:pageview),
-        build(:pageview, domain: site.domain),
+        build(:pageview),
-        build(:pageview, domain: site.domain)
+        build(:pageview)
       ])
 
       perform_job(SendTrialNotifications, %{})
@@ -115,9 +115,9 @@ defmodule Plausible.Workers.SendTrialNotificationsTest do
       site = insert(:site, members: [user])
 
       populate_stats(site, [
-        build(:pageview, domain: site.domain),
+        build(:pageview),
-        build(:pageview, domain: site.domain),
+        build(:pageview),
-        build(:pageview, domain: site.domain)
+        build(:pageview)
       ])
 
       perform_job(SendTrialNotifications, %{})
@@ -130,9 +130,9 @@ defmodule Plausible.Workers.SendTrialNotificationsTest do
       site = insert(:site, members: [user])
 
       populate_stats(site, [
-        build(:pageview, domain: site.domain),
+        build(:pageview),
-        build(:pageview, domain: site.domain),
+        build(:pageview),
-        build(:pageview, domain: site.domain)
+        build(:pageview)
       ])
 
       insert(:subscription, user: user)